From 6b6ed91b92d0564929863ce482177f0545f5d687 Mon Sep 17 00:00:00 2001 From: Justin Visser Date: Fri, 20 Feb 2026 01:28:20 +0100 Subject: [PATCH] modularize service layer, add mobile nav drawer, and sync docs --- README.md | 21 +- agents.md | 11 +- app/api/runtime_deps.py | 4 +- app/main.py | 2 +- app/services/continuation_queue.py | 347 +-- app/services/domains/__init__.py | 1 + app/services/domains/ingestion/__init__.py | 1 + app/services/domains/ingestion/application.py | 2110 +++++++++++++++ app/services/domains/ingestion/constants.py | 31 + .../domains/ingestion/fingerprints.py | 136 + app/services/domains/ingestion/queue.py | 348 +++ app/services/domains/ingestion/safety.py | 282 ++ app/services/domains/ingestion/scheduler.py | 582 +++++ app/services/domains/ingestion/types.py | 108 + app/services/domains/portability/__init__.py | 1 + .../domains/portability/application.py | 67 + app/services/domains/portability/constants.py | 9 + app/services/domains/portability/exporting.py | 91 + app/services/domains/portability/normalize.py | 109 + .../domains/portability/publication_import.py | 368 +++ .../domains/portability/scholar_import.py | 107 + app/services/domains/portability/types.py | 24 + app/services/domains/publications/__init__.py | 1 + .../domains/publications/application.py | 50 + app/services/domains/publications/counts.py | 68 + app/services/domains/publications/listing.py | 89 + app/services/domains/publications/modes.py | 18 + app/services/domains/publications/queries.py | 139 + .../domains/publications/read_state.py | 71 + app/services/domains/publications/types.py | 33 + app/services/domains/runs/__init__.py | 1 + app/services/domains/runs/application.py | 45 + app/services/domains/runs/queue_queries.py | 70 + app/services/domains/runs/queue_service.py | 182 ++ app/services/domains/runs/runs_service.py | 79 + app/services/domains/runs/summary.py | 76 + app/services/domains/runs/types.py | 38 + app/services/domains/scholar/__init__.py | 1 + app/services/domains/scholar/author_rows.py | 202 ++ app/services/domains/scholar/parser.py | 110 + .../domains/scholar/parser_constants.py | 87 + app/services/domains/scholar/parser_types.py | 59 + app/services/domains/scholar/parser_utils.py | 42 + app/services/domains/scholar/profile_rows.py | 324 +++ app/services/domains/scholar/source.py | 225 ++ .../domains/scholar/state_detection.py | 164 ++ app/services/domains/scholars/__init__.py | 1 + app/services/domains/scholars/application.py | 966 +++++++ app/services/domains/scholars/constants.py | 70 + app/services/domains/scholars/exceptions.py | 5 + app/services/domains/scholars/search_hints.py | 67 + app/services/domains/scholars/uploads.py | 39 + app/services/domains/scholars/validators.py | 38 + app/services/domains/settings/__init__.py | 1 + app/services/domains/settings/application.py | 146 ++ app/services/domains/users/__init__.py | 1 + app/services/domains/users/application.py | 98 + app/services/import_export.py | 666 +---- app/services/ingestion.py | 2325 +---------------- app/services/publications.py | 384 +-- app/services/run_safety.py | 281 +- app/services/runs.py | 434 +-- app/services/scheduler.py | 581 +--- app/services/scholar_parser.py | 885 +------ app/services/scholar_source.py | 225 +- app/services/scholars.py | 1122 +------- app/services/user_settings.py | 145 +- app/services/users.py | 97 +- frontend/src/app/AppShell.vue | 52 +- frontend/src/components/layout/AppHeader.vue | 45 + frontend/src/components/layout/AppNav.vue | 9 + frontend/src/pages/ScholarsPage.vue | 6 +- 72 files changed, 8122 insertions(+), 7501 deletions(-) create mode 100644 app/services/domains/__init__.py create mode 100644 app/services/domains/ingestion/__init__.py create mode 100644 app/services/domains/ingestion/application.py create mode 100644 app/services/domains/ingestion/constants.py create mode 100644 app/services/domains/ingestion/fingerprints.py create mode 100644 app/services/domains/ingestion/queue.py create mode 100644 app/services/domains/ingestion/safety.py create mode 100644 app/services/domains/ingestion/scheduler.py create mode 100644 app/services/domains/ingestion/types.py create mode 100644 app/services/domains/portability/__init__.py create mode 100644 app/services/domains/portability/application.py create mode 100644 app/services/domains/portability/constants.py create mode 100644 app/services/domains/portability/exporting.py create mode 100644 app/services/domains/portability/normalize.py create mode 100644 app/services/domains/portability/publication_import.py create mode 100644 app/services/domains/portability/scholar_import.py create mode 100644 app/services/domains/portability/types.py create mode 100644 app/services/domains/publications/__init__.py create mode 100644 app/services/domains/publications/application.py create mode 100644 app/services/domains/publications/counts.py create mode 100644 app/services/domains/publications/listing.py create mode 100644 app/services/domains/publications/modes.py create mode 100644 app/services/domains/publications/queries.py create mode 100644 app/services/domains/publications/read_state.py create mode 100644 app/services/domains/publications/types.py create mode 100644 app/services/domains/runs/__init__.py create mode 100644 app/services/domains/runs/application.py create mode 100644 app/services/domains/runs/queue_queries.py create mode 100644 app/services/domains/runs/queue_service.py create mode 100644 app/services/domains/runs/runs_service.py create mode 100644 app/services/domains/runs/summary.py create mode 100644 app/services/domains/runs/types.py create mode 100644 app/services/domains/scholar/__init__.py create mode 100644 app/services/domains/scholar/author_rows.py create mode 100644 app/services/domains/scholar/parser.py create mode 100644 app/services/domains/scholar/parser_constants.py create mode 100644 app/services/domains/scholar/parser_types.py create mode 100644 app/services/domains/scholar/parser_utils.py create mode 100644 app/services/domains/scholar/profile_rows.py create mode 100644 app/services/domains/scholar/source.py create mode 100644 app/services/domains/scholar/state_detection.py create mode 100644 app/services/domains/scholars/__init__.py create mode 100644 app/services/domains/scholars/application.py create mode 100644 app/services/domains/scholars/constants.py create mode 100644 app/services/domains/scholars/exceptions.py create mode 100644 app/services/domains/scholars/search_hints.py create mode 100644 app/services/domains/scholars/uploads.py create mode 100644 app/services/domains/scholars/validators.py create mode 100644 app/services/domains/settings/__init__.py create mode 100644 app/services/domains/settings/application.py create mode 100644 app/services/domains/users/__init__.py create mode 100644 app/services/domains/users/application.py diff --git a/README.md b/README.md index 752617c..f4f3fca 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Self-hosted scholar tracking with a single app image (API + frontend). -[![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/justinzeus/scholarr/actions/workflows/ci.yml) +[![CI](https://img.shields.io/github/actions/workflow/status/justinzeus/scholarr/ci.yml?style=for-the-badge)](https://github.com/JustinZeus/scholar-scraper/actions/workflows/ci.yml) [![Docker Pulls](https://img.shields.io/docker/pulls/justinzeus/scholarr?style=for-the-badge&logo=docker)](https://hub.docker.com/r/justinzeus/scholarr) [![Docker Image](https://img.shields.io/badge/docker-justinzeus%2Fscholarr-2496ED?style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/justinzeus/scholarr) @@ -82,11 +82,20 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml down ## Backend Architecture (Refactored) -- `app/services/ingestion.py`: run orchestration, page retry/pagination logic, publication upsert/dedup, safety counters, continuation queue sync. -- `app/services/scholar_parser.py`: strict DOM parsing with explicit layout-state detection and fail-fast warning codes for malformed Scholar markup. -- `app/services/scheduler.py`: automatic/scheduled ingestion runner plus continuation queue draining and retry/drop policy. -- `app/services/publications.py`: user-scoped publication listing/read-state update queries used by dashboard/publication UIs. -- `app/services/import_export.py`: JSON import/export for tracked scholars and publication link state. +- Compatibility facades remain in `app/services/*.py`, but canonical service logic now lives in `app/services/domains/*`. +- `app/services/domains/ingestion/*`: run orchestration (`application.py`), continuation queue (`queue.py`), scrape safety cooldown policy (`safety.py`), scheduler/queue draining (`scheduler.py`), plus shared constants/types/fingerprint helpers. +- `app/services/domains/scholar/*`: fail-fast Google Scholar parsing and source access. Parser flow is modularized across row extractors, state detection, constants, and parser types. +- `app/services/domains/scholars/*`: scholar CRUD, name-search safety/caching, profile-image validation/upload handling, and shared scholar validation helpers. +- `app/services/domains/publications/*`: publication listing/read-state query modules for dashboard/publication UIs (`pub_url` + `pdf_url`). +- `app/services/domains/runs/*`: run history summaries plus continuation queue status/retry/drop/clear operations. +- `app/services/domains/portability/*`: import/export of tracked scholars and publication-link state while preserving global publication deduplication. +- `app/services/domains/settings/*` and `app/services/domains/users/*`: user settings and user management services. + +## Frontend Navigation and Layout Notes + +- On mobile (`< lg`), primary navigation is behind a header hamburger and opens as a left-side drawer with backdrop. +- Mobile nav closes on route change, nav link click, and logout. +- Long lists are constrained to internal scroll containers in fill-layout pages (including dashboard recent publications and tracked scholars table view). ## Name Search Status diff --git a/agents.md b/agents.md index 69f864c..b7bd7e3 100644 --- a/agents.md +++ b/agents.md @@ -28,8 +28,9 @@ These limits prevent IP bans and are not to be optimized away. * **Infrastructure:** Multi-stage Docker. ## 5. Refactored Service Boundaries (Current) -* **`app/services/scholar_parser.py`:** Parser contract is fail-fast. Layout drift must emit explicit `layout_*` reasons/warnings, never silent partial success. -* **`app/services/ingestion.py`:** Orchestrates ingestion runs; validate parser outputs before persistence; enforce publication candidate constraints before upsert. -* **`app/services/publications.py`:** Publication list/read-state query layer; include both `pub_url` and `pdf_url` for UI consumption. -* **`app/services/import_export.py`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules. -* **`app/services/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic. +* **Compatibility rule:** `app/services/*.py` files are import façades only. Keep them thin and route all real logic to domain modules. +* **`app/services/domains/scholar/*`:** Parser contract is fail-fast. Layout drift must emit explicit `layout_*` reasons/warnings, never silent partial success. +* **`app/services/domains/ingestion/application.py`:** Orchestrates ingestion runs; validate parser outputs before persistence; enforce publication candidate constraints before upsert. +* **`app/services/domains/publications/*`:** Publication list/read-state query layer; include both `pub_url` and `pdf_url` for UI consumption. +* **`app/services/domains/portability/*`:** Handles JSON import/export for user-scoped scholars and scholar-publication link state while preserving global publication dedup rules. +* **`app/services/domains/ingestion/scheduler.py`:** Owns automatic runs and continuation queue retries/drops; do not bypass safety gate or cooldown logic. diff --git a/app/api/runtime_deps.py b/app/api/runtime_deps.py index 167e2ef..3a692a3 100644 --- a/app/api/runtime_deps.py +++ b/app/api/runtime_deps.py @@ -2,8 +2,8 @@ from __future__ import annotations from fastapi import Depends -from app.services import ingestion as ingestion_service -from app.services import scholar_source as scholar_source_service +from app.services.domains.ingestion import application as ingestion_service +from app.services.domains.scholar import source as scholar_source_service def get_scholar_source() -> scholar_source_service.ScholarSource: diff --git a/app/main.py b/app/main.py index 264a8a2..f8f9c18 100644 --- a/app/main.py +++ b/app/main.py @@ -21,7 +21,7 @@ from app.http.middleware import ( ) from app.logging_config import configure_logging, parse_redact_fields from app.security.csrf import CSRFMiddleware -from app.services.scheduler import SchedulerService +from app.services.domains.ingestion.scheduler import SchedulerService from app.settings import settings logger = logging.getLogger(__name__) diff --git a/app/services/continuation_queue.py b/app/services/continuation_queue.py index 7494420..a41fa8f 100644 --- a/app/services/continuation_queue.py +++ b/app/services/continuation_queue.py @@ -1,348 +1,3 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import IngestionQueueItem, QueueItemStatus - - -@dataclass(frozen=True) -class ContinuationQueueJob: - id: int - user_id: int - scholar_profile_id: int - resume_cstart: int - reason: str - status: str - attempt_count: int - next_attempt_dt: datetime - - -ACTIVE_QUEUE_STATUSES: tuple[str, ...] = ( - QueueItemStatus.QUEUED.value, - QueueItemStatus.RETRYING.value, -) - - -def normalize_cstart(value: int | None) -> int: - if value is None: - return 0 - return max(0, int(value)) - - -def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int: - base = max(1, int(base_seconds)) - attempts = max(1, int(attempt_count)) - maximum = max(base, int(max_seconds)) - seconds = base * (2 ** max(0, attempts - 1)) - return min(seconds, maximum) - - -async def _get_item_for_user_scholar( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, -) -> IngestionQueueItem | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.user_id == user_id, - IngestionQueueItem.scholar_profile_id == scholar_profile_id, - ) - ) - return result.scalar_one_or_none() - - -def _build_queue_item( - *, - now: datetime, - user_id: int, - scholar_profile_id: int, - normalized_cstart: int, - reason: str, - run_id: int | None, - next_attempt_dt: datetime, -) -> IngestionQueueItem: - return IngestionQueueItem( - user_id=user_id, - scholar_profile_id=scholar_profile_id, - resume_cstart=normalized_cstart, - reason=reason, - status=QueueItemStatus.QUEUED.value, - attempt_count=0, - next_attempt_dt=next_attempt_dt, - last_run_id=run_id, - last_error=None, - dropped_reason=None, - dropped_at=None, - created_at=now, - updated_at=now, - ) - - -def _update_existing_queue_item( - *, - item: IngestionQueueItem, - now: datetime, - normalized_cstart: int, - reason: str, - run_id: int | None, - next_attempt_dt: datetime, -) -> None: - item.resume_cstart = normalized_cstart - item.reason = reason - if item.status == QueueItemStatus.DROPPED.value: - item.attempt_count = 0 - item.status = QueueItemStatus.QUEUED.value - item.next_attempt_dt = next_attempt_dt - item.last_run_id = run_id - item.last_error = None - item.dropped_reason = None - item.dropped_at = None - item.updated_at = now - - -async def upsert_job( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, - resume_cstart: int, - reason: str, - run_id: int | None, - delay_seconds: int, -) -> IngestionQueueItem: - now = datetime.now(timezone.utc) - next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds))) - item = await _get_item_for_user_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - ) - normalized_cstart = normalize_cstart(resume_cstart) - if item is None: - item = _build_queue_item( - now=now, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - normalized_cstart=normalized_cstart, - reason=reason, - run_id=run_id, - next_attempt_dt=next_attempt_dt, - ) - db_session.add(item) - return item - - _update_existing_queue_item( - item=item, - now=now, - normalized_cstart=normalized_cstart, - reason=reason, - run_id=run_id, - next_attempt_dt=next_attempt_dt, - ) - return item - - -async def clear_job_for_scholar( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, -) -> bool: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.user_id == user_id, - IngestionQueueItem.scholar_profile_id == scholar_profile_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return False - await db_session.delete(item) - return True - - -async def delete_job_by_id( - db_session: AsyncSession, - *, - job_id: int, -) -> bool: - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return False - await db_session.delete(item) - return True - - -async def list_due_jobs( - db_session: AsyncSession, - *, - now: datetime, - limit: int, -) -> list[ContinuationQueueJob]: - result = await db_session.execute( - select(IngestionQueueItem) - .where( - IngestionQueueItem.next_attempt_dt <= now, - IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES), - ) - .order_by( - IngestionQueueItem.next_attempt_dt.asc(), - IngestionQueueItem.id.asc(), - ) - .limit(limit) - ) - rows = list(result.scalars().all()) - jobs: list[ContinuationQueueJob] = [] - for row in rows: - jobs.append( - ContinuationQueueJob( - id=int(row.id), - user_id=int(row.user_id), - scholar_profile_id=int(row.scholar_profile_id), - resume_cstart=normalize_cstart(row.resume_cstart), - reason=row.reason, - status=row.status, - attempt_count=int(row.attempt_count), - next_attempt_dt=row.next_attempt_dt, - ) - ) - return jobs - - -async def increment_attempt_count( - db_session: AsyncSession, - *, - job_id: int, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.attempt_count = int(item.attempt_count or 0) + 1 - item.updated_at = now - return item - - -async def reset_attempt_count( - db_session: AsyncSession, - *, - job_id: int, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.attempt_count = 0 - item.updated_at = now - return item - - -async def reschedule_job( - db_session: AsyncSession, - *, - job_id: int, - delay_seconds: int, - reason: str, - error: str | None = None, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds))) - item.status = QueueItemStatus.QUEUED.value - item.reason = reason - item.last_error = error - item.dropped_reason = None - item.dropped_at = None - item.updated_at = now - return item - - -async def mark_retrying( - db_session: AsyncSession, - *, - job_id: int, - reason: str | None = None, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status == QueueItemStatus.DROPPED.value: - return item - item.status = QueueItemStatus.RETRYING.value - if reason: - item.reason = reason - item.updated_at = now - return item - - -async def mark_dropped( - db_session: AsyncSession, - *, - job_id: int, - reason: str, - error: str | None = None, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.status = QueueItemStatus.DROPPED.value - item.reason = "dropped" - item.dropped_reason = reason - item.dropped_at = now - if error is not None: - item.last_error = error - item.updated_at = now - return item - - -async def mark_queued_now( - db_session: AsyncSession, - *, - job_id: int, - reason: str, - reset_attempt_count: bool = False, -) -> IngestionQueueItem | None: - now = datetime.now(timezone.utc) - result = await db_session.execute( - select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) - ) - item = result.scalar_one_or_none() - if item is None: - return None - item.status = QueueItemStatus.QUEUED.value - item.reason = reason - item.next_attempt_dt = now - if reset_attempt_count: - item.attempt_count = 0 - item.last_error = None - item.dropped_reason = None - item.dropped_at = None - item.updated_at = now - return item +from app.services.domains.ingestion.queue import * diff --git a/app/services/domains/__init__.py b/app/services/domains/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/app/services/domains/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/app/services/domains/ingestion/__init__.py b/app/services/domains/ingestion/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/app/services/domains/ingestion/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/app/services/domains/ingestion/application.py b/app/services/domains/ingestion/application.py new file mode 100644 index 0000000..5a0fac8 --- /dev/null +++ b/app/services/domains/ingestion/application.py @@ -0,0 +1,2110 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone +import hashlib +import logging +from typing import Any + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + Publication, + RunStatus, + RunTriggerType, + ScholarProfile, + ScholarPublication, +) +from app.services.domains.ingestion.constants import ( + FAILED_STATES, + FAILURE_BUCKET_BLOCKED, + FAILURE_BUCKET_INGESTION, + FAILURE_BUCKET_LAYOUT, + FAILURE_BUCKET_NETWORK, + FAILURE_BUCKET_OTHER, + RESUMABLE_PARTIAL_REASON_PREFIXES, + RESUMABLE_PARTIAL_REASONS, + RUN_LOCK_NAMESPACE, +) +from app.services.domains.ingestion.fingerprints import ( + _build_body_excerpt, + _dedupe_publication_candidates, + _next_cstart_value, + build_initial_page_fingerprint, + build_publication_fingerprint, + build_publication_url, + normalize_title, +) +from app.services.domains.ingestion import queue as queue_service +from app.services.domains.ingestion import safety as run_safety_service +from app.services.domains.ingestion.types import ( + PagedLoopState, + PagedParseResult, + RunAlertSummary, + RunAlreadyInProgressError, + RunBlockedBySafetyPolicyError, + RunExecutionSummary, + RunFailureSummary, + RunProgress, + ScholarProcessingOutcome, +) +from app.services.domains.settings import application as user_settings_service +from app.services.domains.scholar.parser import ( + ParseState, + ParsedProfilePage, + PublicationCandidate, + parse_profile_page, +) +from app.services.domains.scholar.source import FetchResult, ScholarSource +from app.settings import settings + +logger = logging.getLogger(__name__) + + +def _int_or_default(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _classify_failure_bucket(*, state: str, state_reason: str) -> str: + reason = state_reason.strip().lower() + normalized_state = state.strip().lower() + + if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"): + return FAILURE_BUCKET_BLOCKED + if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"): + return FAILURE_BUCKET_NETWORK + if normalized_state == ParseState.LAYOUT_CHANGED.value: + return FAILURE_BUCKET_LAYOUT + if normalized_state == "ingestion_error": + return FAILURE_BUCKET_INGESTION + return FAILURE_BUCKET_OTHER + + +class ScholarIngestionService: + def __init__(self, *, source: ScholarSource) -> None: + self._source = source + + async def _load_user_settings_for_run( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + ): + user_settings = await user_settings_service.get_or_create_settings( + db_session, + user_id=user_id, + ) + await self._enforce_safety_gate( + db_session, + user_settings=user_settings, + user_id=user_id, + trigger_type=trigger_type, + ) + return user_settings + + async def _enforce_safety_gate( + self, + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, + ) -> None: + now_utc = datetime.now(timezone.utc) + previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc) + if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc): + await db_session.commit() + await db_session.refresh(user_settings) + logger.info( + "ingestion.safety_cooldown_cleared", + extra={ + "event": "ingestion.safety_cooldown_cleared", + "user_id": user_id, + "reason": previous.get("cooldown_reason"), + "cooldown_until": previous.get("cooldown_until"), + "metric_name": "ingestion_safety_cooldown_cleared_total", + "metric_value": 1, + }, + ) + now_utc = datetime.now(timezone.utc) + if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc): + await self._raise_safety_blocked_start( + db_session, + user_settings=user_settings, + user_id=user_id, + trigger_type=trigger_type, + now_utc=now_utc, + ) + + async def _raise_safety_blocked_start( + self, + db_session: AsyncSession, + *, + user_settings, + user_id: int, + trigger_type: RunTriggerType, + now_utc: datetime, + ) -> None: + safety_state = run_safety_service.register_cooldown_blocked_start( + user_settings, + now_utc=now_utc, + ) + await db_session.commit() + logger.warning( + "ingestion.safety_policy_blocked_run_start", + extra={ + "event": "ingestion.safety_policy_blocked_run_start", + "user_id": user_id, + "trigger_type": trigger_type.value, + "reason": safety_state.get("cooldown_reason"), + "cooldown_until": safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), + "blocked_start_count": ((safety_state.get("counters") or {}).get("blocked_start_count")), + "metric_name": "ingestion_safety_run_start_blocked_total", + "metric_value": 1, + }, + ) + raise RunBlockedBySafetyPolicyError( + code="scrape_cooldown_active", + message="Scrape safety cooldown is active; run start is temporarily blocked.", + safety_state=safety_state, + ) + + @staticmethod + def _normalize_run_targets( + *, + scholar_profile_ids: set[int] | None, + start_cstart_by_scholar_id: dict[int, int] | None, + ) -> tuple[set[int] | None, dict[int, int]]: + filtered_scholar_ids = ( + {int(value) for value in scholar_profile_ids} + if scholar_profile_ids is not None + else None + ) + start_cstart_map = { + int(key): max(0, int(value)) + for key, value in (start_cstart_by_scholar_id or {}).items() + } + return filtered_scholar_ids, start_cstart_map + + async def _load_target_scholars( + self, + db_session: AsyncSession, + *, + user_id: int, + filtered_scholar_ids: set[int] | None, + ) -> list[ScholarProfile]: + scholars_stmt = ( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id, ScholarProfile.is_enabled.is_(True)) + .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc()) + ) + if filtered_scholar_ids is not None: + scholars_stmt = scholars_stmt.where(ScholarProfile.id.in_(filtered_scholar_ids)) + scholars_result = await db_session.execute(scholars_stmt) + scholars = list(scholars_result.scalars().all()) + await self._clear_missing_filtered_jobs( + db_session, + user_id=user_id, + filtered_scholar_ids=filtered_scholar_ids, + scholars=scholars, + ) + return scholars + + async def _clear_missing_filtered_jobs( + self, + db_session: AsyncSession, + *, + user_id: int, + filtered_scholar_ids: set[int] | None, + scholars: list[ScholarProfile], + ) -> None: + if filtered_scholar_ids is None: + return + found_ids = {int(scholar.id) for scholar in scholars} + missing_ids = filtered_scholar_ids - found_ids + for scholar_profile_id in missing_ids: + await queue_service.clear_job_for_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + ) + + @staticmethod + def _create_running_run( + *, + user_id: int, + trigger_type: RunTriggerType, + scholar_count: int, + idempotency_key: str | None, + ) -> CrawlRun: + return CrawlRun( + user_id=user_id, + trigger_type=trigger_type, + status=RunStatus.RUNNING, + scholar_count=scholar_count, + new_pub_count=0, + idempotency_key=idempotency_key, + error_log={}, + ) + + @staticmethod + def _log_run_started( + *, + user_id: int, + trigger_type: RunTriggerType, + scholar_count: int, + filtered: bool, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> None: + logger.info( + "ingestion.run_started", + extra={ + "event": "ingestion.run_started", + "user_id": user_id, + "trigger_type": trigger_type.value, + "scholar_count": scholar_count, + "is_filtered_run": filtered, + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "max_pages_per_scholar": max_pages_per_scholar, + "page_size": page_size, + "idempotency_key": idempotency_key, + "alert_blocked_failure_threshold": alert_blocked_failure_threshold, + "alert_network_failure_threshold": alert_network_failure_threshold, + "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, + }, + ) + + @staticmethod + async def _wait_between_scholars(*, index: int, request_delay_seconds: int) -> None: + if index <= 0 or request_delay_seconds <= 0: + return + await asyncio.sleep(float(request_delay_seconds)) + + @staticmethod + def _assert_valid_paged_parse_result( + *, + scholar_id: str, + paged_parse_result: PagedParseResult, + ) -> None: + parsed_page = paged_parse_result.parsed_page + if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}: + if any(code.startswith("layout_") for code in parsed_page.warnings): + raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") + for publication in paged_parse_result.publications: + if not publication.title.strip(): + raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") + if publication.citation_count is not None and int(publication.citation_count) < 0: + raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") + + @staticmethod + def _apply_first_page_profile_metadata( + *, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + run_dt: datetime, + ) -> None: + first_page = paged_parse_result.first_page_parsed_page + if first_page.profile_name and not (scholar.display_name or "").strip(): + scholar.display_name = first_page.profile_name + if first_page.profile_image_url: + scholar.profile_image_url = first_page.profile_image_url + if paged_parse_result.first_page_fingerprint_sha256: + scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 + scholar.last_initial_page_checked_at = run_dt + + @staticmethod + def _log_scholar_parsed( + *, + user_id: int, + run_id: int, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + ) -> None: + parsed_page = paged_parse_result.parsed_page + logger.info( + "ingestion.scholar_parsed", + extra={ + "event": "ingestion.scholar_parsed", + "user_id": user_id, + "crawl_run_id": run_id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "publication_count": len(paged_parse_result.publications), + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "warning_count": len(parsed_page.warnings), + "skipped_no_change": paged_parse_result.skipped_no_change, + }, + ) + + @staticmethod + def _build_result_entry( + *, + scholar: ScholarProfile, + start_cstart: int, + paged_parse_result: PagedParseResult, + ) -> dict[str, Any]: + parsed_page = paged_parse_result.parsed_page + return { + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "outcome": "failed", + "attempt_count": len(paged_parse_result.attempt_log), + "publication_count": len(paged_parse_result.publications), + "start_cstart": start_cstart, + "articles_range": parsed_page.articles_range, + "warnings": parsed_page.warnings, + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "continuation_cstart": paged_parse_result.continuation_cstart, + "skipped_no_change": paged_parse_result.skipped_no_change, + "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + } + + def _skipped_no_change_outcome( + self, + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + first_page = paged_parse_result.first_page_parsed_page + scholar.last_run_status = RunStatus.SUCCESS + scholar.last_run_dt = run_dt + result_entry["state"] = first_page.state.value + result_entry["state_reason"] = "no_change_initial_page_signature" + result_entry["outcome"] = "success" + result_entry["publication_count"] = 0 + result_entry["warnings"] = first_page.warnings + result_entry["debug"] = { + "state_reason": "no_change_initial_page_signature", + "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + "attempt_log": paged_parse_result.attempt_log, + "page_logs": paged_parse_result.page_logs, + } + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=0, + discovered_publication_count=0, + ) + + async def _upsert_publications_outcome( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + parsed_page = paged_parse_result.parsed_page + publications = paged_parse_result.publications + had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} + has_partial_set = len(publications) > 0 and had_page_failure + if (not had_page_failure) or has_partial_set: + return await self._upsert_success_or_exception( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_set, + ) + return self._parse_failure_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + async def _upsert_success_or_exception( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, + ) -> ScholarProcessingOutcome: + try: + return await self._upsert_success( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_publication_set, + ) + except Exception as exc: + return self._upsert_exception_outcome( + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + exc=exc, + ) + + async def _upsert_success( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, + ) -> ScholarProcessingOutcome: + discovered_count = await self._upsert_profile_publications( + db_session, + run=run, + scholar=scholar, + publications=paged_parse_result.publications, + ) + is_partial = ( + paged_parse_result.has_more_remaining + or paged_parse_result.pagination_truncated_reason is not None + or has_partial_publication_set + ) + scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS + scholar.last_run_dt = run_dt + result_entry["outcome"] = "partial" if is_partial else "success" + if is_partial: + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=1 if is_partial else 0, + discovered_publication_count=discovered_count, + ) + + def _upsert_exception_outcome( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + exc: Exception, + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["state"] = "ingestion_error" + result_entry["state_reason"] = "publication_upsert_exception" + result_entry["outcome"] = "failed" + result_entry["error"] = str(exc) + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + exception=exc, + ) + logger.exception( + "ingestion.scholar_failed", + extra={ + "event": "ingestion.scholar_failed", + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + def _parse_failure_outcome( + self, + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["debug"] = self._build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + logger.warning( + "ingestion.scholar_parse_failed", + extra={ + "event": "ingestion.scholar_parse_failed", + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": paged_parse_result.parsed_page.state.value, + "state_reason": paged_parse_result.parsed_page.state_reason, + "status_code": paged_parse_result.fetch_result.status_code, + }, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + async def _sync_continuation_queue( + self, + db_session: AsyncSession, + *, + user_id: int, + scholar: ScholarProfile, + run: CrawlRun, + start_cstart: int, + result_entry: dict[str, Any], + paged_parse_result: PagedParseResult, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> None: + queue_reason, queue_cstart = self._resolve_continuation_queue_target( + outcome=str(result_entry.get("outcome", "")), + state=str(result_entry.get("state", "")), + pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, + continuation_cstart=paged_parse_result.continuation_cstart, + fallback_cstart=start_cstart, + ) + if auto_queue_continuations and queue_reason is not None: + await queue_service.upsert_job( + db_session, + user_id=user_id, + scholar_profile_id=scholar.id, + resume_cstart=queue_cstart, + reason=queue_reason, + run_id=run.id, + delay_seconds=queue_delay_seconds, + ) + result_entry["continuation_enqueued"] = True + result_entry["continuation_reason"] = queue_reason + result_entry["continuation_cstart"] = queue_cstart + return + if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id): + result_entry["continuation_cleared"] = True + + async def _process_scholar( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + start_cstart: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> ScholarProcessingOutcome: + try: + return await self._process_scholar_inner( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + start_cstart=start_cstart, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + except Exception as exc: + return self._unexpected_scholar_exception_outcome( + run=run, + scholar=scholar, + start_cstart=start_cstart, + exc=exc, + ) + + async def _process_scholar_inner( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + start_cstart: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> ScholarProcessingOutcome: + run_dt, paged_parse_result, result_entry = await self._fetch_and_prepare_scholar_result( + run=run, + scholar=scholar, + user_id=user_id, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + outcome = await self._resolve_scholar_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + await self._sync_continuation_queue( + db_session, + user_id=user_id, + scholar=scholar, + run=run, + start_cstart=start_cstart, + result_entry=outcome.result_entry, + paged_parse_result=paged_parse_result, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + return outcome + + async def _fetch_and_prepare_scholar_result( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + start_cstart: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + ) -> tuple[datetime, PagedParseResult, dict[str, Any]]: + run_dt = datetime.now(timezone.utc) + paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( + scholar_id=scholar.scholar_id, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages=max_pages_per_scholar, + page_size=page_size, + previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, + ) + self._assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result) + self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt) + self._log_scholar_parsed(user_id=user_id, run_id=run.id, scholar=scholar, paged_parse_result=paged_parse_result) + result_entry = self._build_result_entry( + scholar=scholar, + start_cstart=start_cstart, + paged_parse_result=paged_parse_result, + ) + return run_dt, paged_parse_result, result_entry + + async def _resolve_scholar_outcome( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + ) -> ScholarProcessingOutcome: + if paged_parse_result.skipped_no_change: + return self._skipped_no_change_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + return await self._upsert_publications_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + @staticmethod + def _apply_outcome_to_progress( + *, + progress: RunProgress, + run: CrawlRun, + outcome: ScholarProcessingOutcome, + ) -> None: + progress.succeeded_count += outcome.succeeded_count_delta + progress.failed_count += outcome.failed_count_delta + progress.partial_count += outcome.partial_count_delta + run.new_pub_count = int(run.new_pub_count or 0) + outcome.discovered_publication_count + progress.scholar_results.append(outcome.result_entry) + + def _unexpected_scholar_exception_outcome( + self, + *, + run: CrawlRun, + scholar: ScholarProfile, + start_cstart: int, + exc: Exception, + ) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = datetime.now(timezone.utc) + logger.exception( + "ingestion.scholar_unexpected_failure", + extra={ + "event": "ingestion.scholar_unexpected_failure", + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome( + result_entry={ + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": "ingestion_error", + "state_reason": "scholar_processing_exception", + "outcome": "failed", + "attempt_count": 0, + "publication_count": 0, + "start_cstart": start_cstart, + "warnings": [], + "error": str(exc), + "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, + }, + succeeded_count_delta=0, + failed_count_delta=1, + partial_count_delta=0, + discovered_publication_count=0, + ) + + @staticmethod + def _summarize_failures( + *, + scholar_results: list[dict[str, Any]], + ) -> RunFailureSummary: + failed_state_counts: dict[str, int] = {} + failed_reason_counts: dict[str, int] = {} + scrape_failure_counts: dict[str, int] = {} + retries_scheduled_count = 0 + scholars_with_retries_count = 0 + retry_exhausted_count = 0 + for entry in scholar_results: + retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1) + if retries_for_entry > 0: + retries_scheduled_count += retries_for_entry + scholars_with_retries_count += 1 + if str(entry.get("outcome", "")) != "failed": + continue + state = str(entry.get("state", "")).strip() + if state not in FAILED_STATES: + continue + failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 + reason = str(entry.get("state_reason", "")).strip() + if reason: + failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 + bucket = _classify_failure_bucket(state=state, state_reason=reason) + scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1 + if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0: + retry_exhausted_count += 1 + return RunFailureSummary( + failed_state_counts=failed_state_counts, + failed_reason_counts=failed_reason_counts, + scrape_failure_counts=scrape_failure_counts, + retries_scheduled_count=retries_scheduled_count, + scholars_with_retries_count=scholars_with_retries_count, + retry_exhausted_count=retry_exhausted_count, + ) + + @staticmethod + def _build_alert_summary( + *, + failure_summary: RunFailureSummary, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> RunAlertSummary: + blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) + network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) + blocked_threshold = max(1, int(alert_blocked_failure_threshold)) + network_threshold = max(1, int(alert_network_failure_threshold)) + retry_threshold = max(1, int(alert_retry_scheduled_threshold)) + alert_flags = { + "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold, + "network_failure_threshold_exceeded": network_failure_count >= network_threshold, + "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold, + } + return RunAlertSummary( + blocked_failure_count=blocked_failure_count, + network_failure_count=network_failure_count, + blocked_failure_threshold=blocked_threshold, + network_failure_threshold=network_threshold, + retry_scheduled_threshold=retry_threshold, + alert_flags=alert_flags, + ) + + @staticmethod + def _log_alert_thresholds( + *, + user_id: int, + run_id: int, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, + ) -> None: + if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]: + logger.warning( + "ingestion.alert_blocked_failure_threshold_exceeded", + extra={ + "event": "ingestion.alert_blocked_failure_threshold_exceeded", + "user_id": user_id, + "crawl_run_id": run_id, + "blocked_failure_count": alert_summary.blocked_failure_count, + "threshold": alert_summary.blocked_failure_threshold, + "metric_name": "ingestion_blocked_failure_threshold_exceeded_total", + "metric_value": 1, + }, + ) + if alert_summary.alert_flags["network_failure_threshold_exceeded"]: + logger.warning( + "ingestion.alert_network_failure_threshold_exceeded", + extra={ + "event": "ingestion.alert_network_failure_threshold_exceeded", + "user_id": user_id, + "crawl_run_id": run_id, + "network_failure_count": alert_summary.network_failure_count, + "threshold": alert_summary.network_failure_threshold, + "metric_name": "ingestion_network_failure_threshold_exceeded_total", + "metric_value": 1, + }, + ) + if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]: + logger.warning( + "ingestion.alert_retry_scheduled_threshold_exceeded", + extra={ + "event": "ingestion.alert_retry_scheduled_threshold_exceeded", + "user_id": user_id, + "crawl_run_id": run_id, + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "threshold": alert_summary.retry_scheduled_threshold, + "metric_name": "ingestion_retry_scheduled_threshold_exceeded_total", + "metric_value": 1, + }, + ) + + @staticmethod + def _apply_safety_outcome( + *, + user_settings, + run: CrawlRun, + user_id: int, + alert_summary: RunAlertSummary, + ) -> None: + pre_apply_state = run_safety_service.get_safety_event_context( + user_settings, + now_utc=datetime.now(timezone.utc), + ) + safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome( + user_settings, + run_id=int(run.id), + blocked_failure_count=alert_summary.blocked_failure_count, + network_failure_count=alert_summary.network_failure_count, + blocked_failure_threshold=alert_summary.blocked_failure_threshold, + network_failure_threshold=alert_summary.network_failure_threshold, + blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, + network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, + now_utc=datetime.now(timezone.utc), + ) + ScholarIngestionService._log_safety_transition( + user_id=user_id, + run_id=int(run.id), + alert_summary=alert_summary, + pre_apply_state=pre_apply_state, + safety_state=safety_state, + cooldown_trigger_reason=cooldown_trigger_reason, + ) + + @staticmethod + def _log_safety_transition( + *, + user_id: int, + run_id: int, + alert_summary: RunAlertSummary, + pre_apply_state: dict[str, Any], + safety_state: dict[str, Any], + cooldown_trigger_reason: str | None, + ) -> None: + if cooldown_trigger_reason is not None: + logger.warning( + "ingestion.safety_cooldown_entered", + extra={ + "event": "ingestion.safety_cooldown_entered", + "user_id": user_id, + "crawl_run_id": run_id, + "reason": cooldown_trigger_reason, + "blocked_failure_count": alert_summary.blocked_failure_count, + "network_failure_count": alert_summary.network_failure_count, + "blocked_failure_threshold": alert_summary.blocked_failure_threshold, + "network_failure_threshold": alert_summary.network_failure_threshold, + "cooldown_until": safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), + "safety_counters": safety_state.get("counters", {}), + "metric_name": "ingestion_safety_cooldown_entered_total", + "metric_value": 1, + }, + ) + elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"): + logger.info( + "ingestion.safety_cooldown_cleared", + extra={ + "event": "ingestion.safety_cooldown_cleared", + "user_id": user_id, + "crawl_run_id": run_id, + "reason": pre_apply_state.get("cooldown_reason"), + "cooldown_until": pre_apply_state.get("cooldown_until"), + "metric_name": "ingestion_safety_cooldown_cleared_total", + "metric_value": 1, + }, + ) + + def _finalize_run_record( + self, + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, + idempotency_key: str | None, + ) -> None: + run.end_dt = datetime.now(timezone.utc) + run.status = self._resolve_run_status( + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + ) + run.error_log = { + "scholar_results": progress.scholar_results, + "summary": { + "succeeded_count": progress.succeeded_count, + "failed_count": progress.failed_count, + "partial_count": progress.partial_count, + "failed_state_counts": failure_summary.failed_state_counts, + "failed_reason_counts": failure_summary.failed_reason_counts, + "scrape_failure_counts": failure_summary.scrape_failure_counts, + "retry_counts": { + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "scholars_with_retries_count": failure_summary.scholars_with_retries_count, + "retry_exhausted_count": failure_summary.retry_exhausted_count, + }, + "alert_thresholds": { + "blocked_failure_threshold": alert_summary.blocked_failure_threshold, + "network_failure_threshold": alert_summary.network_failure_threshold, + "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold, + }, + "alert_flags": alert_summary.alert_flags, + }, + "meta": {"idempotency_key": idempotency_key} if idempotency_key else {}, + } + + @staticmethod + def _log_run_completed( + *, + user_id: int, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + alert_summary: RunAlertSummary, + failure_summary: RunFailureSummary, + ) -> None: + logger.info( + "ingestion.run_completed", + extra={ + "event": "ingestion.run_completed", + "user_id": user_id, + "crawl_run_id": run.id, + "status": run.status.value, + "scholar_count": len(scholars), + "succeeded_count": progress.succeeded_count, + "failed_count": progress.failed_count, + "partial_count": progress.partial_count, + "new_publication_count": run.new_pub_count, + "blocked_failure_count": alert_summary.blocked_failure_count, + "network_failure_count": alert_summary.network_failure_count, + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "alert_flags": alert_summary.alert_flags, + }, + ) + + @staticmethod + def _paging_kwargs( + *, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + ) -> dict[str, Any]: + return { + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "max_pages_per_scholar": max_pages_per_scholar, + "page_size": page_size, + } + + @staticmethod + def _threshold_kwargs( + *, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> dict[str, Any]: + return { + "alert_blocked_failure_threshold": alert_blocked_failure_threshold, + "alert_network_failure_threshold": alert_network_failure_threshold, + "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, + } + + @staticmethod + def _run_execution_summary( + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + ) -> RunExecutionSummary: + return RunExecutionSummary( + crawl_run_id=run.id, + status=run.status, + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + new_publication_count=run.new_pub_count, + ) + + async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: + user_settings = await self._load_user_settings_for_run( + db_session, + user_id=user_id, + trigger_type=trigger_type, + ) + if not await self._try_acquire_user_lock(db_session, user_id=user_id): + raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") + filtered_scholar_ids, start_cstart_map = self._normalize_run_targets( + scholar_profile_ids=scholar_profile_ids, + start_cstart_by_scholar_id=start_cstart_by_scholar_id, + ) + scholars = await self._load_target_scholars( + db_session, + user_id=user_id, + filtered_scholar_ids=filtered_scholar_ids, + ) + run = await self._start_run_record_for_targets( + db_session, + user_id=user_id, + trigger_type=trigger_type, + scholars=scholars, + filtered=filtered_scholar_ids is not None, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + return user_settings, run, scholars, start_cstart_map + + async def _start_run_record_for_targets( + self, + db_session: AsyncSession, + *, + user_id: int, + trigger_type: RunTriggerType, + scholars: list[ScholarProfile], + filtered: bool, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> CrawlRun: + self._log_run_started( + user_id=user_id, + trigger_type=trigger_type, + scholar_count=len(scholars), + filtered=filtered, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + idempotency_key=idempotency_key, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + run = self._create_running_run( + user_id=user_id, + trigger_type=trigger_type, + scholar_count=len(scholars), + idempotency_key=idempotency_key, + ) + db_session.add(run) + await db_session.flush() + return run + + async def _run_scholar_iteration( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + start_cstart_map: dict[int, int], + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, + ) -> RunProgress: + progress = RunProgress() + for index, scholar in enumerate(scholars): + await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) + start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) + outcome = await self._process_scholar( + db_session, + run=run, + scholar=scholar, + user_id=user_id, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + start_cstart=start_cstart, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome) + return progress + + def _complete_run_for_user( + self, + *, + user_settings: Any, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + progress: RunProgress, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, + ) -> tuple[RunFailureSummary, RunAlertSummary]: + failure_summary = self._summarize_failures(scholar_results=progress.scholar_results) + alert_summary = self._build_alert_summary( + failure_summary=failure_summary, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + self._log_alert_thresholds( + user_id=user_id, + run_id=int(run.id), + failure_summary=failure_summary, + alert_summary=alert_summary, + ) + self._apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary) + self._finalize_run_record( + run=run, + scholars=scholars, + progress=progress, + failure_summary=failure_summary, + alert_summary=alert_summary, + idempotency_key=idempotency_key, + ) + return failure_summary, alert_summary + + async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: + paging_kwargs = self._paging_kwargs(request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) + threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) + user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( + db_session, + user_id=user_id, + trigger_type=trigger_type, + scholar_profile_ids=scholar_profile_ids, + start_cstart_by_scholar_id=start_cstart_by_scholar_id, + idempotency_key=idempotency_key, + **paging_kwargs, + **threshold_kwargs, + ) + progress = await self._run_scholar_iteration( + db_session, + run=run, + scholars=scholars, + user_id=user_id, + start_cstart_map=start_cstart_map, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + **paging_kwargs, + ) + failure_summary, alert_summary = self._complete_run_for_user( + user_settings=user_settings, + run=run, + scholars=scholars, + user_id=user_id, + progress=progress, + idempotency_key=idempotency_key, + **threshold_kwargs, + ) + await db_session.commit() + self._log_run_completed(user_id=user_id, run=run, scholars=scholars, progress=progress, alert_summary=alert_summary, failure_summary=failure_summary) + return self._run_execution_summary(run=run, scholars=scholars, progress=progress) + + async def _fetch_profile_page( + self, + *, + scholar_id: str, + cstart: int, + page_size: int, + ) -> FetchResult: + try: + page_fetcher = getattr(self._source, "fetch_profile_page_html", None) + if callable(page_fetcher): + return await page_fetcher( + scholar_id, + cstart=cstart, + pagesize=page_size, + ) + if cstart <= 0: + return await self._source.fetch_profile_html(scholar_id) + return FetchResult( + requested_url=( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" + ), + status_code=None, + final_url=None, + body="", + error="source_does_not_support_pagination", + ) + except Exception as exc: + logger.exception( + "ingestion.fetch_unexpected_error", + extra={ + "event": "ingestion.fetch_unexpected_error", + "scholar_id": scholar_id, + "cstart": cstart, + "page_size": page_size, + }, + ) + return FetchResult( + requested_url=( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" + ), + status_code=None, + final_url=None, + body="", + error=str(exc), + ) + + @staticmethod + def _attempt_log_entry( + *, + attempt: int, + cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + ) -> dict[str, Any]: + return { + "attempt": attempt, + "cstart": cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + } + + @staticmethod + def _should_retry_network_page( + *, + parsed_page: ParsedProfilePage, + attempt_index: int, + max_attempts: int, + ) -> bool: + return parsed_page.state == ParseState.NETWORK_ERROR and attempt_index < max_attempts - 1 + + @staticmethod + async def _sleep_retry_backoff( + *, + scholar_id: str, + cstart: int, + attempt_index: int, + backoff: float, + state_reason: str, + ) -> None: + sleep_seconds = backoff * (2**attempt_index) + logger.warning( + "ingestion.scholar_retry_scheduled", + extra={ + "event": "ingestion.scholar_retry_scheduled", + "scholar_id": scholar_id, + "cstart": cstart, + "attempt": attempt_index + 1, + "next_attempt": attempt_index + 2, + "sleep_seconds": sleep_seconds, + "state_reason": state_reason, + }, + ) + if sleep_seconds > 0: + await asyncio.sleep(sleep_seconds) + + async def _fetch_and_parse_page_with_retry( + self, + *, + scholar_id: str, + cstart: int, + page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: + max_attempts = max(1, int(network_error_retries) + 1) + backoff = max(float(retry_backoff_seconds), 0.0) + attempt_log: list[dict[str, Any]] = [] + fetch_result: FetchResult | None = None + parsed_page: ParsedProfilePage | None = None + + for attempt_index in range(max_attempts): + fetch_result = await self._fetch_profile_page( + scholar_id=scholar_id, + cstart=cstart, + page_size=page_size, + ) + parsed_page = parse_profile_page(fetch_result) + attempt_log.append( + self._attempt_log_entry( + attempt=attempt_index + 1, + cstart=cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + ) + ) + if not self._should_retry_network_page( + parsed_page=parsed_page, + attempt_index=attempt_index, + max_attempts=max_attempts, + ): + break + await self._sleep_retry_backoff( + scholar_id=scholar_id, + cstart=cstart, + attempt_index=attempt_index, + backoff=backoff, + state_reason=parsed_page.state_reason, + ) + + if fetch_result is None or parsed_page is None: + raise RuntimeError("Fetch-and-parse retry loop produced no result.") + return fetch_result, parsed_page, attempt_log + + @staticmethod + def _page_log_entry( + *, + page_number: int, + cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_count: int, + ) -> dict[str, Any]: + return { + "page": page_number, + "cstart": cstart, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "status_code": fetch_result.status_code, + "publication_count": len(parsed_page.publications), + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "warning_codes": parsed_page.warnings, + "attempt_count": attempt_count, + } + + @staticmethod + def _should_skip_no_change( + *, + start_cstart: int, + first_page_fingerprint_sha256: str | None, + previous_initial_page_fingerprint_sha256: str | None, + parsed_page: ParsedProfilePage, + ) -> bool: + return ( + start_cstart <= 0 + and first_page_fingerprint_sha256 is not None + and previous_initial_page_fingerprint_sha256 is not None + and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256 + and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} + ) + + @staticmethod + def _skip_no_change_result( + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult: + return PagedParseResult( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=[], + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=1, + pages_attempted=1, + has_more_remaining=False, + pagination_truncated_reason=None, + continuation_cstart=None, + skipped_no_change=True, + ) + + @staticmethod + def _initial_failure_result( + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + start_cstart: int, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult: + continuation_cstart = start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None + return PagedParseResult( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=[], + attempt_log=attempt_log, + page_logs=page_logs, + pages_fetched=0, + pages_attempted=1, + has_more_remaining=False, + pagination_truncated_reason=None, + continuation_cstart=continuation_cstart, + skipped_no_change=False, + ) + + @staticmethod + def _build_loop_state( + *, + start_cstart: int, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedLoopState: + next_cstart = _next_cstart_value( + articles_range=parsed_page.articles_range, + fallback=start_cstart + len(parsed_page.publications), + ) + return PagedLoopState( + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + publications=list(parsed_page.publications), + pages_fetched=1, + pages_attempted=1, + current_cstart=start_cstart, + next_cstart=next_cstart, + ) + + @staticmethod + def _set_truncated_state( + *, + state: PagedLoopState, + reason: str, + continuation_cstart: int, + ) -> None: + state.has_more_remaining = True + state.pagination_truncated_reason = reason + state.continuation_cstart = continuation_cstart + + def _should_stop_pagination(self, *, state: PagedLoopState, bounded_max_pages: int) -> bool: + if state.pages_fetched >= bounded_max_pages: + self._set_truncated_state( + state=state, + reason="max_pages_reached", + continuation_cstart=( + state.next_cstart if state.next_cstart > state.current_cstart else state.current_cstart + ), + ) + return True + if state.next_cstart <= state.current_cstart: + self._set_truncated_state( + state=state, + reason="pagination_cursor_stalled", + continuation_cstart=state.current_cstart, + ) + return True + return False + + async def _fetch_next_page( + self, + *, + scholar_id: str, + state: PagedLoopState, + request_delay_seconds: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: + if request_delay_seconds > 0: + await asyncio.sleep(float(request_delay_seconds)) + state.current_cstart = state.next_cstart + return await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=state.current_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + + @staticmethod + def _record_next_page( + *, + state: PagedLoopState, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + page_attempt_log: list[dict[str, Any]], + ) -> None: + state.pages_attempted += 1 + state.attempt_log.extend(page_attempt_log) + state.page_logs.append( + ScholarIngestionService._page_log_entry( + page_number=state.pages_attempted, + cstart=state.current_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_count=len(page_attempt_log), + ) + ) + state.fetch_result = fetch_result + state.parsed_page = parsed_page + + @staticmethod + def _handle_page_state_transition(*, state: PagedLoopState) -> bool: + if state.parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + ScholarIngestionService._set_truncated_state( + state=state, + reason=f"page_state_{state.parsed_page.state.value}", + continuation_cstart=state.current_cstart, + ) + return True + if state.parsed_page.state == ParseState.NO_RESULTS and len(state.parsed_page.publications) == 0: + state.pages_fetched += 1 + return True + state.pages_fetched += 1 + state.publications.extend(state.parsed_page.publications) + state.next_cstart = _next_cstart_value( + articles_range=state.parsed_page.articles_range, + fallback=state.current_cstart + len(state.parsed_page.publications), + ) + return False + + async def _fetch_initial_page_context( + self, + *, + scholar_id: str, + start_cstart: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]: + fetch_result, parsed_page, first_attempt_log = await self._fetch_and_parse_page_with_retry( + scholar_id=scholar_id, + cstart=start_cstart, + page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page) + attempt_log = list(first_attempt_log) + page_logs = [ + self._page_log_entry( + page_number=1, + cstart=start_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_count=len(first_attempt_log), + ) + ] + return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs + + async def _paginate_loop( + self, + *, + scholar_id: str, + state: PagedLoopState, + bounded_max_pages: int, + request_delay_seconds: int, + bounded_page_size: int, + network_error_retries: int, + retry_backoff_seconds: float, + ) -> None: + while state.parsed_page.has_show_more_button: + if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages): + return + next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page( + scholar_id=scholar_id, + state=state, + request_delay_seconds=request_delay_seconds, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + self._record_next_page( + state=state, + fetch_result=next_fetch_result, + parsed_page=next_parsed_page, + page_attempt_log=next_attempt_log, + ) + if self._handle_page_state_transition(state=state): + return + + @staticmethod + def _result_from_pagination_state( + *, + state: PagedLoopState, + first_page_fetch_result: FetchResult, + first_page_parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + ) -> PagedParseResult: + return PagedParseResult( + fetch_result=state.fetch_result, + parsed_page=state.parsed_page, + first_page_fetch_result=first_page_fetch_result, + first_page_parsed_page=first_page_parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + publications=_dedupe_publication_candidates(state.publications), + attempt_log=state.attempt_log, + page_logs=state.page_logs, + pages_fetched=state.pages_fetched, + pages_attempted=state.pages_attempted, + has_more_remaining=state.has_more_remaining, + pagination_truncated_reason=state.pagination_truncated_reason, + continuation_cstart=state.continuation_cstart, + skipped_no_change=False, + ) + + def _short_circuit_initial_page( + self, + *, + start_cstart: int, + previous_initial_page_fingerprint_sha256: str | None, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + first_page_fingerprint_sha256: str | None, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]], + ) -> PagedParseResult | None: + if self._should_skip_no_change( + start_cstart=start_cstart, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, + parsed_page=parsed_page, + ): + return self._skip_no_change_result( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + attempt_log=attempt_log, + page_logs=page_logs, + ) + if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + return self._initial_failure_result( + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + start_cstart=start_cstart, + attempt_log=attempt_log, + page_logs=page_logs, + ) + return None + + async def _fetch_and_parse_all_pages_with_retry(self, *, scholar_id: str, start_cstart: int, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages: int, page_size: int, previous_initial_page_fingerprint_sha256: str | None = None) -> PagedParseResult: + bounded_max_pages = max(1, int(max_pages)) + bounded_page_size = max(1, int(page_size)) + fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs = ( + await self._fetch_initial_page_context( + scholar_id=scholar_id, + start_cstart=start_cstart, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + )) + shortcut_result = self._short_circuit_initial_page( + start_cstart=start_cstart, + previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, + fetch_result=fetch_result, + parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + attempt_log=attempt_log, + page_logs=page_logs, + ) + if shortcut_result is not None: + return shortcut_result + state = self._build_loop_state( + start_cstart=start_cstart, + fetch_result=fetch_result, + parsed_page=parsed_page, + attempt_log=attempt_log, + page_logs=page_logs, + ) + await self._paginate_loop( + scholar_id=scholar_id, + state=state, + bounded_max_pages=bounded_max_pages, + request_delay_seconds=request_delay_seconds, + bounded_page_size=bounded_page_size, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + return self._result_from_pagination_state( + state=state, + first_page_fetch_result=fetch_result, + first_page_parsed_page=parsed_page, + first_page_fingerprint_sha256=first_page_fingerprint_sha256, + ) + + async def _upsert_profile_publications( + self, + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + publications: list[PublicationCandidate], + ) -> int: + seen_publication_ids: set[int] = set() + discovered_count = 0 + + for candidate in publications: + publication = await self._resolve_publication(db_session, candidate) + if publication.id in seen_publication_ids: + continue + seen_publication_ids.add(publication.id) + + link_result = await db_session.execute( + select(ScholarPublication).where( + ScholarPublication.scholar_profile_id == scholar.id, + ScholarPublication.publication_id == publication.id, + ) + ) + link = link_result.scalar_one_or_none() + if link is not None: + continue + + link = ScholarPublication( + scholar_profile_id=scholar.id, + publication_id=publication.id, + is_read=False, + first_seen_run_id=run.id, + ) + db_session.add(link) + discovered_count += 1 + + logger.debug( + "ingestion.publication_discovered", + extra={ + "event": "ingestion.publication_discovered", + "scholar_profile_id": scholar.id, + "publication_id": publication.id, + "crawl_run_id": run.id, + }, + ) + + if not scholar.baseline_completed: + scholar.baseline_completed = True + + return discovered_count + + @staticmethod + def _validate_publication_candidate(candidate: PublicationCandidate) -> None: + if not candidate.title.strip(): + raise RuntimeError("Publication candidate is missing title.") + if candidate.citation_count is not None and int(candidate.citation_count) < 0: + raise RuntimeError("Publication candidate has negative citation_count.") + + async def _find_publication_by_cluster( + self, + db_session: AsyncSession, + *, + cluster_id: str | None, + ) -> Publication | None: + if not cluster_id: + return None + result = await db_session.execute( + select(Publication).where(Publication.cluster_id == cluster_id) + ) + return result.scalar_one_or_none() + + async def _find_publication_by_fingerprint( + self, + db_session: AsyncSession, + *, + fingerprint: str, + ) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.fingerprint_sha256 == fingerprint) + ) + return result.scalar_one_or_none() + + @staticmethod + def _select_existing_publication( + *, + cluster_publication: Publication | None, + fingerprint_publication: Publication | None, + ) -> Publication | None: + if cluster_publication is not None: + return cluster_publication + return fingerprint_publication + + async def _create_publication( + self, + db_session: AsyncSession, + *, + candidate: PublicationCandidate, + fingerprint: str, + ) -> Publication: + publication = Publication( + cluster_id=candidate.cluster_id, + fingerprint_sha256=fingerprint, + title_raw=candidate.title, + title_normalized=normalize_title(candidate.title), + year=candidate.year, + citation_count=int(candidate.citation_count or 0), + author_text=candidate.authors_text, + venue_text=candidate.venue_text, + pub_url=build_publication_url(candidate.title_url), + pdf_url=build_publication_url(candidate.pdf_url), + ) + db_session.add(publication) + await db_session.flush() + logger.debug( + "ingestion.publication_created", + extra={ + "event": "ingestion.publication_created", + "publication_id": publication.id, + "cluster_id": publication.cluster_id, + }, + ) + return publication + + @staticmethod + def _update_existing_publication( + *, + publication: Publication, + candidate: PublicationCandidate, + ) -> None: + if candidate.cluster_id and publication.cluster_id is None: + publication.cluster_id = candidate.cluster_id + publication.title_raw = candidate.title + publication.title_normalized = normalize_title(candidate.title) + if candidate.year is not None: + publication.year = candidate.year + if candidate.citation_count is not None: + publication.citation_count = int(candidate.citation_count) + if candidate.authors_text: + publication.author_text = candidate.authors_text + if candidate.venue_text: + publication.venue_text = candidate.venue_text + if candidate.title_url: + publication.pub_url = build_publication_url(candidate.title_url) + if candidate.pdf_url: + publication.pdf_url = build_publication_url(candidate.pdf_url) + + async def _resolve_publication( + self, + db_session: AsyncSession, + candidate: PublicationCandidate, + ) -> Publication: + self._validate_publication_candidate(candidate) + fingerprint = build_publication_fingerprint(candidate) + cluster_publication = await self._find_publication_by_cluster( + db_session, + cluster_id=candidate.cluster_id, + ) + fingerprint_publication = await self._find_publication_by_fingerprint( + db_session, + fingerprint=fingerprint, + ) + publication = self._select_existing_publication( + cluster_publication=cluster_publication, + fingerprint_publication=fingerprint_publication, + ) + if publication is None: + return await self._create_publication( + db_session, + candidate=candidate, + fingerprint=fingerprint, + ) + self._update_existing_publication( + publication=publication, + candidate=candidate, + ) + return publication + + def _resolve_run_status( + self, + *, + scholar_count: int, + succeeded_count: int, + failed_count: int, + partial_count: int, + ) -> RunStatus: + if scholar_count == 0: + return RunStatus.SUCCESS + if failed_count == scholar_count: + return RunStatus.FAILED + if failed_count > 0 or partial_count > 0: + return RunStatus.PARTIAL_FAILURE + if succeeded_count > 0: + return RunStatus.SUCCESS + return RunStatus.FAILED + + def _resolve_continuation_queue_target( + self, + *, + outcome: str, + state: str, + pagination_truncated_reason: str | None, + continuation_cstart: int | None, + fallback_cstart: int, + ) -> tuple[str | None, int]: + if outcome == "partial": + reason = (pagination_truncated_reason or "").strip() + if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith( + RESUMABLE_PARTIAL_REASON_PREFIXES + ): + return reason, queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + return None, queue_service.normalize_cstart(fallback_cstart) + + if outcome == "failed" and state == ParseState.NETWORK_ERROR.value: + return "network_error_retry", queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + + return None, queue_service.normalize_cstart(fallback_cstart) + + def _build_failure_debug_context( + self, + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]] | None = None, + exception: Exception | None = None, + ) -> dict[str, Any]: + context: dict[str, Any] = { + "requested_url": fetch_result.requested_url, + "final_url": fetch_result.final_url, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + "state_reason": parsed_page.state_reason, + "profile_name": parsed_page.profile_name, + "profile_image_url": parsed_page.profile_image_url, + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "has_operation_error_banner": parsed_page.has_operation_error_banner, + "warning_codes": parsed_page.warnings, + "marker_counts_nonzero": { + key: value for key, value in parsed_page.marker_counts.items() if value > 0 + }, + "body_length": len(fetch_result.body), + "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() + if fetch_result.body + else None, + "body_excerpt": _build_body_excerpt(fetch_result.body), + "attempt_log": attempt_log, + } + if page_logs: + context["page_logs"] = page_logs + if exception is not None: + context["exception_type"] = type(exception).__name__ + context["exception_message"] = str(exception) + return context + + async def _try_acquire_user_lock( + self, + db_session: AsyncSession, + *, + user_id: int, + ) -> bool: + result = await db_session.execute( + text( + "SELECT pg_try_advisory_xact_lock(:namespace, :user_key)" + ), + { + "namespace": RUN_LOCK_NAMESPACE, + "user_key": int(user_id), + }, + ) + return bool(result.scalar_one()) diff --git a/app/services/domains/ingestion/constants.py b/app/services/domains/ingestion/constants.py new file mode 100644 index 0000000..cfe2ba0 --- /dev/null +++ b/app/services/domains/ingestion/constants.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import re + +from app.services.domains.scholar.parser import ParseState + +TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+") +WORD_RE = re.compile(r"[a-z0-9]+") +HTML_TAG_RE = re.compile(r"<[^>]+>", re.S) +SPACE_RE = re.compile(r"\s+") + +FAILED_STATES = { + ParseState.BLOCKED_OR_CAPTCHA.value, + ParseState.LAYOUT_CHANGED.value, + ParseState.NETWORK_ERROR.value, + "ingestion_error", +} + +FAILURE_BUCKET_BLOCKED = "blocked_or_captcha" +FAILURE_BUCKET_NETWORK = "network_error" +FAILURE_BUCKET_LAYOUT = "layout_changed" +FAILURE_BUCKET_INGESTION = "ingestion_error" +FAILURE_BUCKET_OTHER = "other_failure" + +RUN_LOCK_NAMESPACE = 8217 +RESUMABLE_PARTIAL_REASONS = { + "max_pages_reached", + "pagination_cursor_stalled", +} +RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",) +INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30 diff --git a/app/services/domains/ingestion/fingerprints.py b/app/services/domains/ingestion/fingerprints.py new file mode 100644 index 0000000..cf3e796 --- /dev/null +++ b/app/services/domains/ingestion/fingerprints.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any +from urllib.parse import urljoin + +from app.services.domains.ingestion.constants import ( + HTML_TAG_RE, + INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS, + SPACE_RE, + TITLE_ALNUM_RE, + WORD_RE, +) +from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate + + +def normalize_title(value: str) -> str: + lowered = value.lower() + return TITLE_ALNUM_RE.sub("", lowered) + + +def _first_author_last_name(authors_text: str | None) -> str: + if not authors_text: + return "" + first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() + words = WORD_RE.findall(first_author) + if not words: + return "" + return words[-1] + + +def _first_venue_word(venue_text: str | None) -> str: + if not venue_text: + return "" + words = WORD_RE.findall(venue_text.lower()) + if not words: + return "" + return words[0] + + +def build_publication_fingerprint(candidate: PublicationCandidate) -> str: + canonical = "|".join( + [ + normalize_title(candidate.title), + str(candidate.year) if candidate.year is not None else "", + _first_author_last_name(candidate.authors_text), + _first_venue_word(candidate.venue_text), + ] + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None: + if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: + return None + + normalized_rows: list[dict[str, Any]] = [] + for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]: + normalized_rows.append( + { + "cluster_id": publication.cluster_id or "", + "title_normalized": normalize_title(publication.title), + "year": publication.year, + "citation_count": publication.citation_count, + } + ) + + payload = { + "state": parsed_page.state.value, + "articles_range": parsed_page.articles_range or "", + "has_show_more_button": parsed_page.has_show_more_button, + "profile_name": parsed_page.profile_name or "", + "publications": normalized_rows, + } + canonical = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def build_publication_url(path_or_url: str | None) -> str | None: + if not path_or_url: + return None + return urljoin("https://scholar.google.com", path_or_url) + + +def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int: + if articles_range: + numbers = re.findall(r"\d+", articles_range) + if len(numbers) >= 2: + try: + return int(numbers[1]) + except ValueError: + pass + return int(fallback) + + +def _dedupe_publication_candidates( + publications: list[PublicationCandidate], +) -> list[PublicationCandidate]: + deduped: list[PublicationCandidate] = [] + seen: set[str] = set() + for publication in publications: + if publication.cluster_id: + identity = f"cluster:{publication.cluster_id}" + else: + identity = "|".join( + [ + "fallback", + normalize_title(publication.title), + str(publication.year) if publication.year is not None else "", + publication.authors_text or "", + publication.venue_text or "", + ] + ) + if identity in seen: + continue + seen.add(identity) + deduped.append(publication) + return deduped + + +def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None: + if not body: + return None + flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip() + if not flattened: + return None + if len(flattened) <= max_chars: + return flattened + return f"{flattened[:max_chars - 1]}..." diff --git a/app/services/domains/ingestion/queue.py b/app/services/domains/ingestion/queue.py new file mode 100644 index 0000000..7494420 --- /dev/null +++ b/app/services/domains/ingestion/queue.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import IngestionQueueItem, QueueItemStatus + + +@dataclass(frozen=True) +class ContinuationQueueJob: + id: int + user_id: int + scholar_profile_id: int + resume_cstart: int + reason: str + status: str + attempt_count: int + next_attempt_dt: datetime + + +ACTIVE_QUEUE_STATUSES: tuple[str, ...] = ( + QueueItemStatus.QUEUED.value, + QueueItemStatus.RETRYING.value, +) + + +def normalize_cstart(value: int | None) -> int: + if value is None: + return 0 + return max(0, int(value)) + + +def compute_backoff_seconds(*, base_seconds: int, attempt_count: int, max_seconds: int) -> int: + base = max(1, int(base_seconds)) + attempts = max(1, int(attempt_count)) + maximum = max(base, int(max_seconds)) + seconds = base * (2 ** max(0, attempts - 1)) + return min(seconds, maximum) + + +async def _get_item_for_user_scholar( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> IngestionQueueItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.scholar_profile_id == scholar_profile_id, + ) + ) + return result.scalar_one_or_none() + + +def _build_queue_item( + *, + now: datetime, + user_id: int, + scholar_profile_id: int, + normalized_cstart: int, + reason: str, + run_id: int | None, + next_attempt_dt: datetime, +) -> IngestionQueueItem: + return IngestionQueueItem( + user_id=user_id, + scholar_profile_id=scholar_profile_id, + resume_cstart=normalized_cstart, + reason=reason, + status=QueueItemStatus.QUEUED.value, + attempt_count=0, + next_attempt_dt=next_attempt_dt, + last_run_id=run_id, + last_error=None, + dropped_reason=None, + dropped_at=None, + created_at=now, + updated_at=now, + ) + + +def _update_existing_queue_item( + *, + item: IngestionQueueItem, + now: datetime, + normalized_cstart: int, + reason: str, + run_id: int | None, + next_attempt_dt: datetime, +) -> None: + item.resume_cstart = normalized_cstart + item.reason = reason + if item.status == QueueItemStatus.DROPPED.value: + item.attempt_count = 0 + item.status = QueueItemStatus.QUEUED.value + item.next_attempt_dt = next_attempt_dt + item.last_run_id = run_id + item.last_error = None + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + + +async def upsert_job( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, + resume_cstart: int, + reason: str, + run_id: int | None, + delay_seconds: int, +) -> IngestionQueueItem: + now = datetime.now(timezone.utc) + next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds))) + item = await _get_item_for_user_scholar( + db_session, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + ) + normalized_cstart = normalize_cstart(resume_cstart) + if item is None: + item = _build_queue_item( + now=now, + user_id=user_id, + scholar_profile_id=scholar_profile_id, + normalized_cstart=normalized_cstart, + reason=reason, + run_id=run_id, + next_attempt_dt=next_attempt_dt, + ) + db_session.add(item) + return item + + _update_existing_queue_item( + item=item, + now=now, + normalized_cstart=normalized_cstart, + reason=reason, + run_id=run_id, + next_attempt_dt=next_attempt_dt, + ) + return item + + +async def clear_job_for_scholar( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> bool: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.user_id == user_id, + IngestionQueueItem.scholar_profile_id == scholar_profile_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return False + await db_session.delete(item) + return True + + +async def delete_job_by_id( + db_session: AsyncSession, + *, + job_id: int, +) -> bool: + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return False + await db_session.delete(item) + return True + + +async def list_due_jobs( + db_session: AsyncSession, + *, + now: datetime, + limit: int, +) -> list[ContinuationQueueJob]: + result = await db_session.execute( + select(IngestionQueueItem) + .where( + IngestionQueueItem.next_attempt_dt <= now, + IngestionQueueItem.status.in_(ACTIVE_QUEUE_STATUSES), + ) + .order_by( + IngestionQueueItem.next_attempt_dt.asc(), + IngestionQueueItem.id.asc(), + ) + .limit(limit) + ) + rows = list(result.scalars().all()) + jobs: list[ContinuationQueueJob] = [] + for row in rows: + jobs.append( + ContinuationQueueJob( + id=int(row.id), + user_id=int(row.user_id), + scholar_profile_id=int(row.scholar_profile_id), + resume_cstart=normalize_cstart(row.resume_cstart), + reason=row.reason, + status=row.status, + attempt_count=int(row.attempt_count), + next_attempt_dt=row.next_attempt_dt, + ) + ) + return jobs + + +async def increment_attempt_count( + db_session: AsyncSession, + *, + job_id: int, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.attempt_count = int(item.attempt_count or 0) + 1 + item.updated_at = now + return item + + +async def reset_attempt_count( + db_session: AsyncSession, + *, + job_id: int, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.attempt_count = 0 + item.updated_at = now + return item + + +async def reschedule_job( + db_session: AsyncSession, + *, + job_id: int, + delay_seconds: int, + reason: str, + error: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.next_attempt_dt = now + timedelta(seconds=max(1, int(delay_seconds))) + item.status = QueueItemStatus.QUEUED.value + item.reason = reason + item.last_error = error + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + return item + + +async def mark_retrying( + db_session: AsyncSession, + *, + job_id: int, + reason: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QueueItemStatus.DROPPED.value: + return item + item.status = QueueItemStatus.RETRYING.value + if reason: + item.reason = reason + item.updated_at = now + return item + + +async def mark_dropped( + db_session: AsyncSession, + *, + job_id: int, + reason: str, + error: str | None = None, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.status = QueueItemStatus.DROPPED.value + item.reason = "dropped" + item.dropped_reason = reason + item.dropped_at = now + if error is not None: + item.last_error = error + item.updated_at = now + return item + + +async def mark_queued_now( + db_session: AsyncSession, + *, + job_id: int, + reason: str, + reset_attempt_count: bool = False, +) -> IngestionQueueItem | None: + now = datetime.now(timezone.utc) + result = await db_session.execute( + select(IngestionQueueItem).where(IngestionQueueItem.id == job_id) + ) + item = result.scalar_one_or_none() + if item is None: + return None + item.status = QueueItemStatus.QUEUED.value + item.reason = reason + item.next_attempt_dt = now + if reset_attempt_count: + item.attempt_count = 0 + item.last_error = None + item.dropped_reason = None + item.dropped_at = None + item.updated_at = now + return item diff --git a/app/services/domains/ingestion/safety.py b/app/services/domains/ingestion/safety.py new file mode 100644 index 0000000..debb8c5 --- /dev/null +++ b/app/services/domains/ingestion/safety.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any + +from app.db.models import UserSetting + +COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded" +COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded" + +_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs" +_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs" +_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count" +_COUNTER_BLOCKED_START_COUNT = "blocked_start_count" +_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count" +_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count" +_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id" + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def _safe_int(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _safe_optional_int(value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _normalize_datetime(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def _state_dict(settings: UserSetting) -> dict[str, Any]: + state = settings.scrape_safety_state + if isinstance(state, dict): + return state + return {} + + +def _counters_from_state(settings: UserSetting) -> dict[str, Any]: + state = _state_dict(settings) + return { + _COUNTER_CONSECUTIVE_BLOCKED_RUNS: max( + 0, + _safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0), + ), + _COUNTER_CONSECUTIVE_NETWORK_RUNS: max( + 0, + _safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0), + ), + _COUNTER_COOLDOWN_ENTRY_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0), + ), + _COUNTER_BLOCKED_START_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0), + ), + _COUNTER_LAST_BLOCKED_FAILURE_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0), + ), + _COUNTER_LAST_NETWORK_FAILURE_COUNT: max( + 0, + _safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0), + ), + _COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int( + state.get(_COUNTER_LAST_EVALUATED_RUN_ID), + ), + } + + +def _cooldown_reason_label(reason: str | None) -> str | None: + if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: + return "Blocked responses exceeded safety threshold" + if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: + return "Network failures exceeded safety threshold" + return None + + +def _recommended_action(reason: str | None) -> str | None: + if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: + return ( + "Google Scholar appears to be blocking requests. Wait for cooldown to expire, " + "increase request delay, and avoid repeated manual retries." + ) + if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: + return ( + "Network failures crossed the threshold. Verify connectivity and retry after cooldown." + ) + return None + + +def is_cooldown_active( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> bool: + now = now_utc or _utcnow() + cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) + if cooldown_until is None: + return False + return cooldown_until > now + + +def clear_expired_cooldown( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> bool: + now = now_utc or _utcnow() + cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) + if cooldown_until is None: + return False + if cooldown_until > now: + return False + settings.scrape_cooldown_until = None + settings.scrape_cooldown_reason = None + return True + + +def register_cooldown_blocked_start( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> dict[str, Any]: + now = now_utc or _utcnow() + counters = _counters_from_state(settings) + counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1 + settings.scrape_safety_state = counters + return get_safety_state_payload(settings, now_utc=now) + + +def _update_run_counters( + *, + counters: dict[str, Any], + run_id: int, + blocked_failure_count: int, + network_failure_count: int, +) -> tuple[int, int]: + bounded_blocked_failures = max(0, int(blocked_failure_count)) + bounded_network_failures = max(0, int(network_failure_count)) + counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures + counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures + counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id) + counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = ( + int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 + if bounded_blocked_failures > 0 + else 0 + ) + counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = ( + int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 + if bounded_network_failures > 0 + else 0 + ) + return bounded_blocked_failures, bounded_network_failures + + +def _resolve_cooldown_trigger( + *, + blocked_failures: int, + network_failures: int, + blocked_failure_threshold: int, + network_failure_threshold: int, + blocked_cooldown_seconds: int, + network_cooldown_seconds: int, +) -> tuple[str | None, int]: + if blocked_failures >= max(1, int(blocked_failure_threshold)): + return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds)) + if network_failures >= max(1, int(network_failure_threshold)): + return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds)) + return None, 0 + + +def _apply_cooldown_decision( + *, + settings: UserSetting, + counters: dict[str, Any], + now: datetime, + reason: str | None, + cooldown_seconds: int, +) -> None: + if reason is None: + clear_expired_cooldown(settings, now_utc=now) + return + settings.scrape_cooldown_reason = reason + settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds))) + counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1 + + +def apply_run_safety_outcome( + settings: UserSetting, + *, + run_id: int, + blocked_failure_count: int, + network_failure_count: int, + blocked_failure_threshold: int, + network_failure_threshold: int, + blocked_cooldown_seconds: int, + network_cooldown_seconds: int, + now_utc: datetime | None = None, +) -> tuple[dict[str, Any], str | None]: + now = now_utc or _utcnow() + counters = _counters_from_state(settings) + blocked_failures, network_failures = _update_run_counters( + counters=counters, + run_id=run_id, + blocked_failure_count=blocked_failure_count, + network_failure_count=network_failure_count, + ) + reason, cooldown_seconds = _resolve_cooldown_trigger( + blocked_failures=blocked_failures, + network_failures=network_failures, + blocked_failure_threshold=blocked_failure_threshold, + network_failure_threshold=network_failure_threshold, + blocked_cooldown_seconds=blocked_cooldown_seconds, + network_cooldown_seconds=network_cooldown_seconds, + ) + _apply_cooldown_decision( + settings=settings, + counters=counters, + now=now, + reason=reason, + cooldown_seconds=cooldown_seconds, + ) + settings.scrape_safety_state = counters + return get_safety_state_payload(settings, now_utc=now), reason + + +def get_safety_state_payload( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> dict[str, Any]: + now = now_utc or _utcnow() + cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) + cooldown_active = bool(cooldown_until is not None and cooldown_until > now) + cooldown_remaining_seconds = 0 + if cooldown_active and cooldown_until is not None: + cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds())) + + reason = settings.scrape_cooldown_reason if cooldown_active else None + + return { + "cooldown_active": cooldown_active, + "cooldown_reason": reason, + "cooldown_reason_label": _cooldown_reason_label(reason), + "cooldown_until": cooldown_until, + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "recommended_action": _recommended_action(reason), + "counters": _counters_from_state(settings), + } + + +def get_safety_event_context( + settings: UserSetting, + *, + now_utc: datetime | None = None, +) -> dict[str, Any]: + payload = get_safety_state_payload(settings, now_utc=now_utc) + return { + "cooldown_active": bool(payload.get("cooldown_active")), + "cooldown_reason": payload.get("cooldown_reason"), + "cooldown_until": payload.get("cooldown_until"), + "cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0), + "safety_counters": payload.get("counters", {}), + } diff --git a/app/services/domains/ingestion/scheduler.py b/app/services/domains/ingestion/scheduler.py new file mode 100644 index 0000000..2959b63 --- /dev/null +++ b/app/services/domains/ingestion/scheduler.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +import logging + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + QueueItemStatus, + RunTriggerType, + ScholarProfile, + User, + UserSetting, +) +from app.db.session import get_session_factory +from app.services.domains.ingestion import queue as queue_service +from app.services.domains.ingestion.application import ( + RunAlreadyInProgressError, + RunBlockedBySafetyPolicyError, + ScholarIngestionService, +) +from app.services.domains.scholar.source import LiveScholarSource +from app.settings import settings + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class _AutoRunCandidate: + user_id: int + run_interval_minutes: int + request_delay_seconds: int + cooldown_until: datetime | None + cooldown_reason: str | None + + +class SchedulerService: + def __init__( + self, + *, + enabled: bool, + tick_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + continuation_queue_enabled: bool, + continuation_base_delay_seconds: int, + continuation_max_delay_seconds: int, + continuation_max_attempts: int, + queue_batch_size: int, + ) -> None: + self._enabled = enabled + self._tick_seconds = max(5, int(tick_seconds)) + self._network_error_retries = max(0, int(network_error_retries)) + self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds)) + self._max_pages_per_scholar = max(1, int(max_pages_per_scholar)) + self._page_size = max(1, int(page_size)) + self._continuation_queue_enabled = bool(continuation_queue_enabled) + self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds)) + self._continuation_max_delay_seconds = max( + self._continuation_base_delay_seconds, + int(continuation_max_delay_seconds), + ) + self._continuation_max_attempts = max(1, int(continuation_max_attempts)) + self._queue_batch_size = max(1, int(queue_batch_size)) + self._task: asyncio.Task[None] | None = None + self._source = LiveScholarSource() + + async def start(self) -> None: + if not self._enabled: + logger.info( + "scheduler.disabled", + extra={ + "event": "scheduler.disabled", + }, + ) + return + if self._task is not None: + return + self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler") + logger.info( + "scheduler.started", + extra={ + "event": "scheduler.started", + "tick_seconds": self._tick_seconds, + "network_error_retries": self._network_error_retries, + "retry_backoff_seconds": self._retry_backoff_seconds, + "max_pages_per_scholar": self._max_pages_per_scholar, + "page_size": self._page_size, + "continuation_queue_enabled": self._continuation_queue_enabled, + "continuation_base_delay_seconds": self._continuation_base_delay_seconds, + "continuation_max_delay_seconds": self._continuation_max_delay_seconds, + "continuation_max_attempts": self._continuation_max_attempts, + "queue_batch_size": self._queue_batch_size, + }, + ) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + finally: + self._task = None + logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"}) + + async def _run_loop(self) -> None: + while True: + try: + await self._tick_once() + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "scheduler.tick_failed", + extra={ + "event": "scheduler.tick_failed", + }, + ) + await asyncio.sleep(float(self._tick_seconds)) + + async def _tick_once(self) -> None: + if self._continuation_queue_enabled: + await self._drain_continuation_queue() + candidates = await self._load_candidates() + if not candidates: + return + now = datetime.now(timezone.utc) + for candidate in candidates: + if not await self._is_due(candidate, now=now): + continue + await self._run_candidate(candidate) + + async def _load_candidate_rows(self) -> list[tuple]: + session_factory = get_session_factory() + async with session_factory() as session: + result = await session.execute( + select( + UserSetting.user_id, + UserSetting.run_interval_minutes, + UserSetting.request_delay_seconds, + UserSetting.scrape_cooldown_until, + UserSetting.scrape_cooldown_reason, + ) + .join(User, User.id == UserSetting.user_id) + .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True)) + .order_by(UserSetting.user_id.asc()) + ) + return result.all() + + @staticmethod + def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None: + user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row + if cooldown_until is not None and cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + if cooldown_until is not None and cooldown_until > now_utc: + logger.info( + "scheduler.run_skipped_safety_cooldown_precheck", + extra={ + "event": "scheduler.run_skipped_safety_cooldown_precheck", + "user_id": int(user_id), + "reason": cooldown_reason, + "cooldown_until": cooldown_until, + "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()), + "metric_name": "scheduler_run_skipped_safety_cooldown_total", + "metric_value": 1, + }, + ) + return None + return _AutoRunCandidate( + user_id=int(user_id), + run_interval_minutes=int(run_interval_minutes), + request_delay_seconds=int(request_delay_seconds), + cooldown_until=cooldown_until, + cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), + ) + + async def _load_candidates(self) -> list[_AutoRunCandidate]: + if not settings.ingestion_automation_allowed: + return [] + rows = await self._load_candidate_rows() + now_utc = datetime.now(timezone.utc) + candidates: list[_AutoRunCandidate] = [] + for row in rows: + candidate = self._candidate_from_row(row, now_utc=now_utc) + if candidate is not None: + candidates.append(candidate) + return candidates + + async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool: + session_factory = get_session_factory() + async with session_factory() as session: + result = await session.execute( + select(CrawlRun.start_dt) + .where( + CrawlRun.user_id == candidate.user_id, + ) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(1) + ) + last_run = result.scalar_one_or_none() + + if last_run is None: + return True + + next_due_dt = last_run + timedelta( + minutes=candidate.run_interval_minutes + ) + return now >= next_due_dt + + async def _run_candidate_ingestion( + self, + *, + candidate: _AutoRunCandidate, + ): + session_factory = get_session_factory() + async with session_factory() as session: + ingestion = ScholarIngestionService(source=self._source) + try: + return await ingestion.run_for_user( + session, + user_id=candidate.user_id, + trigger_type=RunTriggerType.SCHEDULED, + request_delay_seconds=candidate.request_delay_seconds, + network_error_retries=self._network_error_retries, + retry_backoff_seconds=self._retry_backoff_seconds, + max_pages_per_scholar=self._max_pages_per_scholar, + page_size=self._page_size, + auto_queue_continuations=self._continuation_queue_enabled, + queue_delay_seconds=self._continuation_base_delay_seconds, + alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, + alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, + alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, + ) + except RunAlreadyInProgressError: + await session.rollback() + logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id}) + return None + except RunBlockedBySafetyPolicyError as exc: + await session.rollback() + logger.info( + "scheduler.run_skipped_safety_cooldown", + extra={ + "event": "scheduler.run_skipped_safety_cooldown", + "user_id": candidate.user_id, + "reason": exc.safety_state.get("cooldown_reason"), + "cooldown_until": exc.safety_state.get("cooldown_until"), + "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"), + "metric_name": "scheduler_run_skipped_safety_cooldown_total", + "metric_value": 1, + }, + ) + return None + except Exception: + await session.rollback() + logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id}) + return None + + async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: + run_summary = await self._run_candidate_ingestion(candidate=candidate) + if run_summary is None: + return + logger.info( + "scheduler.run_completed", + extra={ + "event": "scheduler.run_completed", + "user_id": candidate.user_id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + "scholar_count": run_summary.scholar_count, + "new_publication_count": run_summary.new_publication_count, + }, + ) + + async def _drain_continuation_queue(self) -> None: + now = datetime.now(timezone.utc) + session_factory = get_session_factory() + async with session_factory() as session: + jobs = await queue_service.list_due_jobs( + session, + now=now, + limit=self._queue_batch_size, + ) + for job in jobs: + await self._run_queue_job(job) + + async def _drop_queue_job_if_max_attempts( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + if job.attempt_count < self._continuation_max_attempts: + return False + session_factory = get_session_factory() + async with session_factory() as session: + dropped = await queue_service.mark_dropped( + session, + job_id=job.id, + reason="max_attempts_reached", + ) + await session.commit() + if dropped is not None: + logger.warning( + "scheduler.queue_item_dropped_max_attempts", + extra={ + "event": "scheduler.queue_item_dropped_max_attempts", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + "attempt_count": job.attempt_count, + "max_attempts": self._continuation_max_attempts, + }, + ) + return True + + async def _mark_queue_job_retrying( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + session_factory = get_session_factory() + async with session_factory() as session: + queue_item = await queue_service.mark_retrying(session, job_id=job.id) + await session.commit() + if queue_item is None: + return False + if queue_item.status == QueueItemStatus.DROPPED.value: + return False + return True + + async def _queue_job_has_available_scholar( + self, + job: queue_service.ContinuationQueueJob, + ) -> bool: + session_factory = get_session_factory() + async with session_factory() as session: + scholar_result = await session.execute( + select(ScholarProfile.id).where( + ScholarProfile.user_id == job.user_id, + ScholarProfile.id == job.scholar_profile_id, + ScholarProfile.is_enabled.is_(True), + ) + ) + scholar_id = scholar_result.scalar_one_or_none() + if scholar_id is not None: + return True + dropped = await queue_service.mark_dropped( + session, + job_id=job.id, + reason="scholar_unavailable", + ) + await session.commit() + if dropped is not None: + logger.info( + "scheduler.queue_item_dropped_scholar_unavailable", + extra={ + "event": "scheduler.queue_item_dropped_scholar_unavailable", + "queue_item_id": job.id, + "user_id": job.user_id, + "scholar_profile_id": job.scholar_profile_id, + }, + ) + return False + + async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None: + session_factory = get_session_factory() + async with session_factory() as recovery_session: + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=max(self._tick_seconds, 15), + reason="user_run_lock_active", + error="run_already_in_progress", + ) + await recovery_session.commit() + logger.info( + "scheduler.queue_item_deferred_lock", + extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id}, + ) + + async def _reschedule_queue_job_safety_cooldown( + self, + job: queue_service.ContinuationQueueJob, + exc: RunBlockedBySafetyPolicyError, + ) -> None: + cooldown_remaining_seconds = max( + self._tick_seconds, + int(exc.safety_state.get("cooldown_remaining_seconds") or 0), + ) + session_factory = get_session_factory() + async with session_factory() as recovery_session: + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds), + reason="scrape_safety_cooldown", + error=str(exc.message), + ) + await recovery_session.commit() + logger.info( + "scheduler.queue_item_deferred_safety_cooldown", + extra={ + "event": "scheduler.queue_item_deferred_safety_cooldown", + "queue_item_id": job.id, + "user_id": job.user_id, + "reason": exc.safety_state.get("cooldown_reason"), + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total", + "metric_value": 1, + }, + ) + + async def _reschedule_queue_job_after_exception( + self, + job: queue_service.ContinuationQueueJob, + *, + exc: Exception, + ) -> None: + session_factory = get_session_factory() + async with session_factory() as recovery_session: + queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id) + if queue_item is None: + await recovery_session.commit() + return + if int(queue_item.attempt_count) >= self._continuation_max_attempts: + await queue_service.mark_dropped( + recovery_session, + job_id=job.id, + reason="scheduler_exception_max_attempts", + error=str(exc), + ) + await recovery_session.commit() + logger.warning( + "scheduler.queue_item_dropped_after_exception", + extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count}, + ) + return + delay_seconds = queue_service.compute_backoff_seconds( + base_seconds=self._continuation_base_delay_seconds, + attempt_count=int(queue_item.attempt_count), + max_seconds=self._continuation_max_delay_seconds, + ) + await queue_service.reschedule_job( + recovery_session, + job_id=job.id, + delay_seconds=delay_seconds, + reason="scheduler_exception", + error=str(exc), + ) + await recovery_session.commit() + logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id}) + + async def _run_ingestion_for_queue_job( + self, + job: queue_service.ContinuationQueueJob, + ): + session_factory = get_session_factory() + async with session_factory() as session: + request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id) + ingestion = ScholarIngestionService(source=self._source) + try: + return await ingestion.run_for_user( + session, + user_id=job.user_id, + trigger_type=RunTriggerType.SCHEDULED, + request_delay_seconds=request_delay_seconds, + network_error_retries=self._network_error_retries, + retry_backoff_seconds=self._retry_backoff_seconds, + max_pages_per_scholar=self._max_pages_per_scholar, + page_size=self._page_size, + scholar_profile_ids={job.scholar_profile_id}, + start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart}, + auto_queue_continuations=self._continuation_queue_enabled, + queue_delay_seconds=self._continuation_base_delay_seconds, + alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, + alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, + alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, + ) + except RunAlreadyInProgressError: + await session.rollback() + await self._reschedule_queue_job_lock_active(job) + except RunBlockedBySafetyPolicyError as exc: + await session.rollback() + await self._reschedule_queue_job_safety_cooldown(job, exc) + except Exception as exc: + await session.rollback() + await self._reschedule_queue_job_after_exception(job, exc=exc) + return None + + @staticmethod + def _log_queue_item_resolved( + *, + event_name: str, + job: queue_service.ContinuationQueueJob, + run_summary, + attempt_count: int | None = None, + delay_seconds: int | None = None, + ) -> None: + payload = { + "event": event_name, + "queue_item_id": job.id, + "user_id": job.user_id, + "run_id": run_summary.crawl_run_id, + "status": run_summary.status.value, + } + if attempt_count is not None: + payload["attempt_count"] = attempt_count + if delay_seconds is not None: + payload["delay_seconds"] = delay_seconds + logger.info(event_name, extra=payload) + + async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None: + session_factory = get_session_factory() + async with session_factory() as session: + if int(run_summary.failed_count) <= 0: + queue_item = await queue_service.reset_attempt_count(session, job_id=job.id) + await session.commit() + if queue_item is None: + self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) + return + self._log_queue_item_resolved( + event_name="scheduler.queue_item_progressed", + job=job, + run_summary=run_summary, + attempt_count=int(queue_item.attempt_count), + ) + return + queue_item = await queue_service.increment_attempt_count(session, job_id=job.id) + if queue_item is None: + await session.commit() + self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) + return + if int(queue_item.attempt_count) >= self._continuation_max_attempts: + await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run") + await session.commit() + logger.warning( + "scheduler.queue_item_dropped_max_attempts_after_run", + extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value}, + ) + return + delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds) + await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error) + await session.commit() + self._log_queue_item_resolved( + event_name="scheduler.queue_item_rescheduled_failed", + job=job, + run_summary=run_summary, + attempt_count=int(queue_item.attempt_count), + delay_seconds=delay_seconds, + ) + + async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None: + if await self._drop_queue_job_if_max_attempts(job): + return + if not await self._mark_queue_job_retrying(job): + return + if not await self._queue_job_has_available_scholar(job): + return + run_summary = await self._run_ingestion_for_queue_job(job) + if run_summary is None: + return + await self._finalize_queue_job_after_run(job, run_summary) + + async def _load_request_delay_for_user( + self, + db_session: AsyncSession, + *, + user_id: int, + ) -> int: + result = await db_session.execute( + select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) + ) + delay = result.scalar_one_or_none() + if delay is None: + return 10 + return max(1, int(delay)) diff --git a/app/services/domains/ingestion/types.py b/app/services/domains/ingestion/types.py new file mode 100644 index 0000000..f6de578 --- /dev/null +++ b/app/services/domains/ingestion/types.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from app.db.models import RunStatus +from app.services.domains.scholar.parser import ParsedProfilePage, PublicationCandidate +from app.services.domains.scholar.source import FetchResult + + +@dataclass(frozen=True) +class RunExecutionSummary: + crawl_run_id: int + status: RunStatus + scholar_count: int + succeeded_count: int + failed_count: int + partial_count: int + new_publication_count: int + + +@dataclass(frozen=True) +class PagedParseResult: + fetch_result: FetchResult + parsed_page: ParsedProfilePage + first_page_fetch_result: FetchResult + first_page_parsed_page: ParsedProfilePage + first_page_fingerprint_sha256: str | None + publications: list[PublicationCandidate] + attempt_log: list[dict[str, Any]] + page_logs: list[dict[str, Any]] + pages_fetched: int + pages_attempted: int + has_more_remaining: bool + pagination_truncated_reason: str | None + continuation_cstart: int | None + skipped_no_change: bool + + +@dataclass +class RunProgress: + succeeded_count: int = 0 + failed_count: int = 0 + partial_count: int = 0 + scholar_results: list[dict[str, Any]] = field(default_factory=list) + + +@dataclass(frozen=True) +class ScholarProcessingOutcome: + result_entry: dict[str, Any] + succeeded_count_delta: int + failed_count_delta: int + partial_count_delta: int + discovered_publication_count: int + + +@dataclass(frozen=True) +class RunFailureSummary: + failed_state_counts: dict[str, int] + failed_reason_counts: dict[str, int] + scrape_failure_counts: dict[str, int] + retries_scheduled_count: int + scholars_with_retries_count: int + retry_exhausted_count: int + + +@dataclass(frozen=True) +class RunAlertSummary: + blocked_failure_count: int + network_failure_count: int + blocked_failure_threshold: int + network_failure_threshold: int + retry_scheduled_threshold: int + alert_flags: dict[str, bool] + + +@dataclass +class PagedLoopState: + fetch_result: FetchResult + parsed_page: ParsedProfilePage + attempt_log: list[dict[str, Any]] + page_logs: list[dict[str, Any]] + publications: list[PublicationCandidate] + pages_fetched: int + pages_attempted: int + current_cstart: int + next_cstart: int + has_more_remaining: bool = False + pagination_truncated_reason: str | None = None + continuation_cstart: int | None = None + + +class RunAlreadyInProgressError(RuntimeError): + """Raised when a run lock for a user is already held by another process.""" + + +class RunBlockedBySafetyPolicyError(RuntimeError): + def __init__( + self, + *, + code: str, + message: str, + safety_state: dict[str, Any], + ) -> None: + super().__init__(message) + self.code = code + self.message = message + self.safety_state = safety_state diff --git a/app/services/domains/portability/__init__.py b/app/services/domains/portability/__init__.py new file mode 100644 index 0000000..28aae8b --- /dev/null +++ b/app/services/domains/portability/__init__.py @@ -0,0 +1 @@ +from app.services.domains.portability.application import * diff --git a/app/services/domains/portability/application.py b/app/services/domains/portability/application.py new file mode 100644 index 0000000..7e0b56e --- /dev/null +++ b/app/services/domains/portability/application.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication +from app.services.domains.portability.constants import ( + EXPORT_SCHEMA_VERSION, + MAX_IMPORT_PUBLICATIONS, + MAX_IMPORT_SCHOLARS, +) +from app.services.domains.portability.exporting import export_user_data +from app.services.domains.portability.normalize import _validate_import_sizes +from app.services.domains.portability.publication_import import ( + _build_imported_publication_input, + _initialize_import_counters, + _upsert_imported_publication, +) +from app.services.domains.portability.scholar_import import _upsert_imported_scholars +from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput + + +async def import_user_data( + db_session: AsyncSession, + *, + user_id: int, + scholars: list[dict[str, Any]], + publications: list[dict[str, Any]], +) -> dict[str, int]: + _validate_import_sizes(scholars=scholars, publications=publications) + scholar_map, counters = await _upsert_imported_scholars( + db_session, + user_id=user_id, + scholars=scholars, + ) + cluster_cache: dict[str, Publication | None] = {} + fingerprint_cache: dict[str, Publication | None] = {} + _initialize_import_counters(counters) + for item in publications: + parsed_item = _build_imported_publication_input( + item=item, + scholar_map=scholar_map, + ) + if parsed_item is None: + counters["skipped_records"] += 1 + continue + await _upsert_imported_publication( + db_session, + payload=parsed_item, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + counters=counters, + ) + await db_session.commit() + return counters + + +__all__ = [ + "EXPORT_SCHEMA_VERSION", + "MAX_IMPORT_SCHOLARS", + "MAX_IMPORT_PUBLICATIONS", + "ImportExportError", + "ImportedPublicationInput", + "export_user_data", + "import_user_data", +] diff --git a/app/services/domains/portability/constants.py b/app/services/domains/portability/constants.py new file mode 100644 index 0000000..6815347 --- /dev/null +++ b/app/services/domains/portability/constants.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import re + +EXPORT_SCHEMA_VERSION = 1 +MAX_IMPORT_SCHOLARS = 10_000 +MAX_IMPORT_PUBLICATIONS = 100_000 +WORD_RE = re.compile(r"[a-z0-9]+") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") diff --git a/app/services/domains/portability/exporting.py b/app/services/domains/portability/exporting.py new file mode 100644 index 0000000..5c89006 --- /dev/null +++ b/app/services/domains/portability/exporting.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, ScholarProfile, ScholarPublication +from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION + + +def _exported_at_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]: + return { + "scholar_id": profile.scholar_id, + "display_name": profile.display_name, + "is_enabled": bool(profile.is_enabled), + "profile_image_override_url": profile.profile_image_override_url, + } + + +def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]: + ( + scholar_id, + cluster_id, + fingerprint_sha256, + title_raw, + year, + citation_count, + author_text, + venue_text, + pub_url, + pdf_url, + is_read, + ) = row + return { + "scholar_id": scholar_id, + "cluster_id": cluster_id, + "fingerprint_sha256": fingerprint_sha256, + "title": title_raw, + "year": year, + "citation_count": int(citation_count or 0), + "author_text": author_text, + "venue_text": venue_text, + "pub_url": pub_url, + "pdf_url": pdf_url, + "is_read": bool(is_read), + } + + +async def export_user_data( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, Any]: + scholars_result = await db_session.execute( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarProfile.id.asc()) + ) + publication_result = await db_session.execute( + select( + ScholarProfile.scholar_id, + Publication.cluster_id, + Publication.fingerprint_sha256, + Publication.title_raw, + Publication.year, + Publication.citation_count, + Publication.author_text, + Publication.venue_text, + Publication.pub_url, + Publication.pdf_url, + ScholarPublication.is_read, + ) + .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id) + .join(Publication, Publication.id == ScholarPublication.publication_id) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + ) + scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()] + publications = [_serialize_export_publication(row) for row in publication_result.all()] + return { + "schema_version": EXPORT_SCHEMA_VERSION, + "exported_at": _exported_at_iso(), + "scholars": scholars, + "publications": publications, + } diff --git a/app/services/domains/portability/normalize.py b/app/services/domains/portability/normalize.py new file mode 100644 index 0000000..60dd619 --- /dev/null +++ b/app/services/domains/portability/normalize.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import hashlib +from typing import Any + +from app.services.domains.ingestion.application import normalize_title +from app.services.domains.portability.constants import ( + MAX_IMPORT_PUBLICATIONS, + MAX_IMPORT_SCHOLARS, + SHA256_RE, + WORD_RE, +) +from app.services.domains.portability.types import ImportExportError + + +def _normalize_optional_text(value: Any) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _normalize_optional_year(value: Any) -> int | None: + if value is None: + return None + try: + year = int(value) + except (TypeError, ValueError): + return None + if year < 1500 or year > 3000: + return None + return year + + +def _normalize_citation_count(value: Any) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + return 0 + return max(0, parsed) + + +def _first_author_last_name(authors_text: str | None) -> str: + if not authors_text: + return "" + first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() + words = WORD_RE.findall(first_author) + if not words: + return "" + return words[-1] + + +def _first_venue_word(venue_text: str | None) -> str: + if not venue_text: + return "" + words = WORD_RE.findall(venue_text.lower()) + if not words: + return "" + return words[0] + + +def _build_fingerprint( + *, + title: str, + year: int | None, + author_text: str | None, + venue_text: str | None, +) -> str: + canonical = "|".join( + [ + normalize_title(title), + str(year) if year is not None else "", + _first_author_last_name(author_text), + _first_venue_word(venue_text), + ] + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _resolve_fingerprint( + *, + title: str, + year: int | None, + author_text: str | None, + venue_text: str | None, + provided_fingerprint: Any, +) -> str: + normalized = _normalize_optional_text(provided_fingerprint) + if normalized and SHA256_RE.fullmatch(normalized.lower()): + return normalized.lower() + return _build_fingerprint( + title=title, + year=year, + author_text=author_text, + venue_text=venue_text, + ) + + +def _validate_import_sizes( + *, + scholars: list[dict[str, Any]], + publications: list[dict[str, Any]], +) -> None: + if len(scholars) > MAX_IMPORT_SCHOLARS: + raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).") + if len(publications) > MAX_IMPORT_PUBLICATIONS: + raise ImportExportError( + f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})." + ) diff --git a/app/services/domains/portability/publication_import.py b/app/services/domains/portability/publication_import.py new file mode 100644 index 0000000..62f81ab --- /dev/null +++ b/app/services/domains/portability/publication_import.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Publication, ScholarProfile, ScholarPublication +from app.services.domains.ingestion.application import build_publication_url, normalize_title +from app.services.domains.portability.normalize import ( + _normalize_citation_count, + _normalize_optional_text, + _normalize_optional_year, + _resolve_fingerprint, +) +from app.services.domains.portability.types import ImportedPublicationInput + + +async def _find_publication_by_cluster( + db_session: AsyncSession, + *, + cluster_id: str, +) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.cluster_id == cluster_id) + ) + return result.scalar_one_or_none() + + +async def _find_publication_by_fingerprint( + db_session: AsyncSession, + *, + fingerprint_sha256: str, +) -> Publication | None: + result = await db_session.execute( + select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256) + ) + return result.scalar_one_or_none() + + +async def _find_linked_publication_by_title( + db_session: AsyncSession, + *, + scholar_profile_id: int, + title: str, +) -> Publication | None: + normalized_title = normalize_title(title) + result = await db_session.execute( + select(Publication) + .join( + ScholarPublication, + ScholarPublication.publication_id == Publication.id, + ) + .where( + ScholarPublication.scholar_profile_id == scholar_profile_id, + Publication.title_normalized == normalized_title, + ) + .order_by(Publication.id.asc()) + .limit(1) + ) + return result.scalar_one_or_none() + + +def _apply_imported_publication_values( + *, + publication: Publication, + title: str, + year: int | None, + citation_count: int, + author_text: str | None, + venue_text: str | None, + pub_url: str | None, + pdf_url: str | None, + cluster_id: str | None, +) -> bool: + updated = False + if cluster_id and publication.cluster_id != cluster_id: + publication.cluster_id = cluster_id + updated = True + if publication.title_raw != title: + publication.title_raw = title + publication.title_normalized = normalize_title(title) + updated = True + if publication.year != year: + publication.year = year + updated = True + if int(publication.citation_count or 0) != citation_count: + publication.citation_count = citation_count + updated = True + if publication.author_text != author_text: + publication.author_text = author_text + updated = True + if publication.venue_text != venue_text: + publication.venue_text = venue_text + updated = True + if pub_url and publication.pub_url != pub_url: + publication.pub_url = pub_url + updated = True + if pdf_url and publication.pdf_url != pdf_url: + publication.pdf_url = pdf_url + updated = True + return updated + + +def _new_publication( + *, + cluster_id: str | None, + fingerprint_sha256: str, + title: str, + year: int | None, + citation_count: int, + author_text: str | None, + venue_text: str | None, + pub_url: str | None, + pdf_url: str | None, +) -> Publication: + return Publication( + cluster_id=cluster_id, + fingerprint_sha256=fingerprint_sha256, + title_raw=title, + title_normalized=normalize_title(title), + year=year, + citation_count=citation_count, + author_text=author_text, + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + ) + + +async def _resolve_publication_for_import( + db_session: AsyncSession, + *, + scholar_profile_id: int, + title: str, + cluster_id: str | None, + fingerprint_sha256: str, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> Publication | None: + if cluster_id: + if cluster_id not in cluster_cache: + cluster_cache[cluster_id] = await _find_publication_by_cluster( + db_session, + cluster_id=cluster_id, + ) + if cluster_cache[cluster_id] is not None: + return cluster_cache[cluster_id] + if fingerprint_sha256 not in fingerprint_cache: + fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint( + db_session, + fingerprint_sha256=fingerprint_sha256, + ) + if fingerprint_cache[fingerprint_sha256] is not None: + return fingerprint_cache[fingerprint_sha256] + return await _find_linked_publication_by_title( + db_session, + scholar_profile_id=scholar_profile_id, + title=title, + ) + + +async def _upsert_scholar_publication_link( + db_session: AsyncSession, + *, + scholar_profile_id: int, + publication_id: int, + is_read: bool, +) -> tuple[bool, bool]: + result = await db_session.execute( + select(ScholarPublication).where( + ScholarPublication.scholar_profile_id == scholar_profile_id, + ScholarPublication.publication_id == publication_id, + ) + ) + link = result.scalar_one_or_none() + if link is None: + db_session.add( + ScholarPublication( + scholar_profile_id=scholar_profile_id, + publication_id=publication_id, + is_read=bool(is_read), + ) + ) + return True, False + if bool(link.is_read) == bool(is_read): + return False, False + link.is_read = bool(is_read) + return False, True + + +def _initialize_import_counters(counters: dict[str, int]) -> None: + counters.update( + { + "publications_created": 0, + "publications_updated": 0, + "links_created": 0, + "links_updated": 0, + } + ) + + +def _build_imported_publication_input( + *, + item: dict[str, Any], + scholar_map: dict[str, ScholarProfile], +) -> ImportedPublicationInput | None: + scholar_id = _normalize_optional_text(item.get("scholar_id")) + title = _normalize_optional_text(item.get("title")) + if not scholar_id or not title: + return None + + profile = scholar_map.get(scholar_id) + if profile is None: + return None + + year = _normalize_optional_year(item.get("year")) + author_text = _normalize_optional_text(item.get("author_text")) + venue_text = _normalize_optional_text(item.get("venue_text")) + return ImportedPublicationInput( + profile=profile, + title=title, + year=year, + citation_count=_normalize_citation_count(item.get("citation_count")), + author_text=author_text, + venue_text=venue_text, + cluster_id=_normalize_optional_text(item.get("cluster_id")), + pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))), + pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))), + fingerprint=_resolve_fingerprint( + title=title, + year=year, + author_text=author_text, + venue_text=venue_text, + provided_fingerprint=item.get("fingerprint_sha256"), + ), + is_read=bool(item.get("is_read", False)), + ) + + +def _update_link_counters( + *, + counters: dict[str, int], + link_created: bool, + link_updated: bool, +) -> None: + if link_created: + counters["links_created"] += 1 + if link_updated: + counters["links_updated"] += 1 + + +def _cache_resolved_publication( + *, + publication: Publication, + cluster_id: str | None, + fingerprint_sha256: str, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> None: + if cluster_id: + cluster_cache[cluster_id] = publication + fingerprint_cache[fingerprint_sha256] = publication + + +async def _create_import_publication( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, +) -> Publication: + publication = _new_publication( + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + title=payload.title, + year=payload.year, + citation_count=payload.citation_count, + author_text=payload.author_text, + venue_text=payload.venue_text, + pub_url=payload.pub_url, + pdf_url=payload.pdf_url, + ) + db_session.add(publication) + await db_session.flush() + return publication + + +def _update_import_publication( + *, + publication: Publication, + payload: ImportedPublicationInput, +) -> bool: + return _apply_imported_publication_values( + publication=publication, + title=payload.title, + year=payload.year, + citation_count=payload.citation_count, + author_text=payload.author_text, + venue_text=payload.venue_text, + pub_url=payload.pub_url, + pdf_url=payload.pdf_url, + cluster_id=payload.cluster_id, + ) + + +async def _upsert_publication_entity( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], +) -> tuple[Publication, bool, bool]: + publication = await _resolve_publication_for_import( + db_session, + scholar_profile_id=int(payload.profile.id), + title=payload.title, + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + created = False + updated = False + if publication is None: + publication = await _create_import_publication(db_session, payload=payload) + created = True + else: + updated = _update_import_publication(publication=publication, payload=payload) + + _cache_resolved_publication( + publication=publication, + cluster_id=payload.cluster_id, + fingerprint_sha256=payload.fingerprint, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + return publication, created, updated + + +async def _upsert_imported_publication( + db_session: AsyncSession, + *, + payload: ImportedPublicationInput, + cluster_cache: dict[str, Publication | None], + fingerprint_cache: dict[str, Publication | None], + counters: dict[str, int], +) -> None: + publication, created, updated = await _upsert_publication_entity( + db_session, + payload=payload, + cluster_cache=cluster_cache, + fingerprint_cache=fingerprint_cache, + ) + if created: + counters["publications_created"] += 1 + if updated: + counters["publications_updated"] += 1 + + link_created, link_updated = await _upsert_scholar_publication_link( + db_session, + scholar_profile_id=int(payload.profile.id), + publication_id=int(publication.id), + is_read=payload.is_read, + ) + _update_link_counters( + counters=counters, + link_created=link_created, + link_updated=link_updated, + ) diff --git a/app/services/domains/portability/scholar_import.py b/app/services/domains/portability/scholar_import.py new file mode 100644 index 0000000..74e6ec1 --- /dev/null +++ b/app/services/domains/portability/scholar_import.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ScholarProfile +from app.services.domains.portability.normalize import _normalize_optional_text +from app.services.domains.scholars import application as scholar_service + + +async def _load_user_scholar_map( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, ScholarProfile]: + result = await db_session.execute( + select(ScholarProfile).where(ScholarProfile.user_id == user_id) + ) + profiles = list(result.scalars().all()) + return {profile.scholar_id: profile for profile in profiles} + + +def _apply_imported_scholar_values( + *, + profile: ScholarProfile, + display_name: str | None, + profile_image_override_url: str | None, + is_enabled: bool, +) -> bool: + updated = False + if display_name and profile.display_name != display_name: + profile.display_name = display_name + updated = True + if profile.profile_image_override_url != profile_image_override_url: + profile.profile_image_override_url = profile_image_override_url + updated = True + if bool(profile.is_enabled) != bool(is_enabled): + profile.is_enabled = bool(is_enabled) + updated = True + return updated + + +def _new_scholar_profile( + *, + user_id: int, + scholar_id: str, + display_name: str | None, + profile_image_override_url: str | None, + is_enabled: bool, +) -> ScholarProfile: + return ScholarProfile( + user_id=user_id, + scholar_id=scholar_id, + display_name=display_name, + profile_image_override_url=profile_image_override_url, + is_enabled=bool(is_enabled), + ) + + +async def _upsert_imported_scholars( + db_session: AsyncSession, + *, + user_id: int, + scholars: list[dict[str, Any]], +) -> tuple[dict[str, ScholarProfile], dict[str, int]]: + scholar_map = await _load_user_scholar_map(db_session, user_id=user_id) + counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0} + for item in scholars: + try: + scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"])) + display_name = scholar_service.normalize_display_name( + str(item.get("display_name") or "") + ) + override_url = scholar_service.normalize_profile_image_url( + _normalize_optional_text(item.get("profile_image_override_url")) + ) + except (KeyError, scholar_service.ScholarServiceError): + counters["skipped_records"] += 1 + continue + + is_enabled = bool(item.get("is_enabled", True)) + existing = scholar_map.get(scholar_id) + if existing is None: + profile = _new_scholar_profile( + user_id=user_id, + scholar_id=scholar_id, + display_name=display_name, + profile_image_override_url=override_url, + is_enabled=is_enabled, + ) + db_session.add(profile) + scholar_map[scholar_id] = profile + counters["scholars_created"] += 1 + continue + + if _apply_imported_scholar_values( + profile=existing, + display_name=display_name, + profile_image_override_url=override_url, + is_enabled=is_enabled, + ): + counters["scholars_updated"] += 1 + + await db_session.flush() + return scholar_map, counters diff --git a/app/services/domains/portability/types.py b/app/services/domains/portability/types.py new file mode 100644 index 0000000..8851b13 --- /dev/null +++ b/app/services/domains/portability/types.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from app.db.models import ScholarProfile + + +class ImportExportError(ValueError): + """Raised when import/export payload constraints are violated.""" + + +@dataclass(frozen=True) +class ImportedPublicationInput: + profile: ScholarProfile + title: str + year: int | None + citation_count: int + author_text: str | None + venue_text: str | None + cluster_id: str | None + pub_url: str | None + pdf_url: str | None + fingerprint: str + is_read: bool diff --git a/app/services/domains/publications/__init__.py b/app/services/domains/publications/__init__.py new file mode 100644 index 0000000..371b02c --- /dev/null +++ b/app/services/domains/publications/__init__.py @@ -0,0 +1 @@ +from app.services.domains.publications.application import * diff --git a/app/services/domains/publications/application.py b/app/services/domains/publications/application.py new file mode 100644 index 0000000..7570de3 --- /dev/null +++ b/app/services/domains/publications/application.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from app.services.domains.publications.counts import ( + count_for_user, + count_latest_for_user, + count_unread_for_user, +) +from app.services.domains.publications.listing import ( + list_for_user, + list_new_for_latest_run_for_user, + list_unread_for_user, +) +from app.services.domains.publications.modes import ( + MODE_ALL, + MODE_LATEST, + MODE_NEW, + MODE_UNREAD, + resolve_mode, + resolve_publication_view_mode, +) +from app.services.domains.publications.queries import ( + get_latest_completed_run_id_for_user, + publications_query, +) +from app.services.domains.publications.read_state import ( + mark_all_unread_as_read_for_user, + mark_selected_as_read_for_user, +) +from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + +__all__ = [ + "MODE_ALL", + "MODE_UNREAD", + "MODE_LATEST", + "MODE_NEW", + "PublicationListItem", + "UnreadPublicationItem", + "resolve_publication_view_mode", + "resolve_mode", + "get_latest_completed_run_id_for_user", + "publications_query", + "list_for_user", + "list_unread_for_user", + "list_new_for_latest_run_for_user", + "count_for_user", + "count_unread_for_user", + "count_latest_for_user", + "mark_all_unread_as_read_for_user", + "mark_selected_as_read_for_user", +] diff --git a/app/services/domains/publications/counts.py b/app/services/domains/publications/counts.py new file mode 100644 index 0000000..98aa57d --- /dev/null +++ b/app/services/domains/publications/counts.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ScholarProfile, ScholarPublication +from app.services.domains.publications.modes import ( + MODE_ALL, + MODE_LATEST, + MODE_UNREAD, + resolve_publication_view_mode, +) +from app.services.domains.publications.queries import get_latest_completed_run_id_for_user + + +async def count_for_user( + db_session: AsyncSession, + *, + user_id: int, + mode: str = MODE_ALL, + scholar_profile_id: int | None = None, +) -> int: + resolved_mode = resolve_publication_view_mode(mode) + latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + stmt = ( + select(func.count()) + .select_from(ScholarPublication) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarProfile.user_id == user_id) + ) + if scholar_profile_id is not None: + stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if resolved_mode == MODE_UNREAD: + stmt = stmt.where(ScholarPublication.is_read.is_(False)) + if resolved_mode == MODE_LATEST: + if latest_run_id is None: + return 0 + stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) + result = await db_session.execute(stmt) + return int(result.scalar_one() or 0) + + +async def count_unread_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int | None = None, +) -> int: + return await count_for_user( + db_session, + user_id=user_id, + mode=MODE_UNREAD, + scholar_profile_id=scholar_profile_id, + ) + + +async def count_latest_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int | None = None, +) -> int: + return await count_for_user( + db_session, + user_id=user_id, + mode=MODE_LATEST, + scholar_profile_id=scholar_profile_id, + ) diff --git a/app/services/domains/publications/listing.py b/app/services/domains/publications/listing.py new file mode 100644 index 0000000..751ccd2 --- /dev/null +++ b/app/services/domains/publications/listing.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.services.domains.publications.modes import ( + MODE_ALL, + MODE_LATEST, + MODE_UNREAD, + resolve_publication_view_mode, +) +from app.services.domains.publications.queries import ( + get_latest_completed_run_id_for_user, + publication_list_item_from_row, + publications_query, + unread_item_from_row, +) +from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + + +async def list_for_user( + db_session: AsyncSession, + *, + user_id: int, + mode: str = MODE_ALL, + scholar_profile_id: int | None = None, + limit: int = 300, +) -> list[PublicationListItem]: + resolved_mode = resolve_publication_view_mode(mode) + latest_run_id = await get_latest_completed_run_id_for_user(db_session, user_id=user_id) + result = await db_session.execute( + publications_query( + user_id=user_id, + mode=resolved_mode, + latest_run_id=latest_run_id, + scholar_profile_id=scholar_profile_id, + limit=limit, + ) + ) + return [ + publication_list_item_from_row(row, latest_run_id=latest_run_id) + for row in result.all() + ] + + +async def list_unread_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, +) -> list[UnreadPublicationItem]: + result = await db_session.execute( + publications_query( + user_id=user_id, + mode=MODE_UNREAD, + latest_run_id=None, + scholar_profile_id=None, + limit=limit, + ) + ) + return [unread_item_from_row(row) for row in result.all()] + + +async def list_new_for_latest_run_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, +) -> list[UnreadPublicationItem]: + rows = await list_for_user( + db_session, + user_id=user_id, + mode=MODE_LATEST, + scholar_profile_id=None, + limit=limit, + ) + return [ + UnreadPublicationItem( + publication_id=row.publication_id, + scholar_profile_id=row.scholar_profile_id, + scholar_label=row.scholar_label, + title=row.title, + year=row.year, + citation_count=row.citation_count, + venue_text=row.venue_text, + pub_url=row.pub_url, + pdf_url=row.pdf_url, + ) + for row in rows + ] diff --git a/app/services/domains/publications/modes.py b/app/services/domains/publications/modes.py new file mode 100644 index 0000000..33629b4 --- /dev/null +++ b/app/services/domains/publications/modes.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +MODE_ALL = "all" +MODE_UNREAD = "unread" +MODE_LATEST = "latest" +MODE_NEW = "new" # compatibility alias for MODE_LATEST + + +def resolve_publication_view_mode(value: str | None) -> str: + if value == MODE_UNREAD: + return MODE_UNREAD + if value in {MODE_LATEST, MODE_NEW}: + return MODE_LATEST + return MODE_ALL + + +def resolve_mode(value: str | None) -> str: + return resolve_publication_view_mode(value) diff --git a/app/services/domains/publications/queries.py b/app/services/domains/publications/queries.py new file mode 100644 index 0000000..1c878df --- /dev/null +++ b/app/services/domains/publications/queries.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from sqlalchemy import Select, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import CrawlRun, Publication, RunStatus, ScholarProfile, ScholarPublication +from app.services.domains.publications.modes import MODE_LATEST, MODE_UNREAD +from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem + + +def _normalized_citation_count(value: object) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +async def get_latest_completed_run_id_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> int | None: + result = await db_session.execute( + select(func.max(CrawlRun.id)).where( + CrawlRun.user_id == user_id, + CrawlRun.status != RunStatus.RUNNING, + ) + ) + latest_run_id = result.scalar_one_or_none() + return int(latest_run_id) if latest_run_id is not None else None + + +def publications_query( + *, + user_id: int, + mode: str, + latest_run_id: int | None, + scholar_profile_id: int | None, + limit: int, +) -> Select[tuple]: + scholar_label = ScholarProfile.display_name + stmt = ( + select( + Publication.id, + ScholarProfile.id, + scholar_label, + ScholarProfile.scholar_id, + Publication.title_raw, + Publication.year, + Publication.citation_count, + Publication.venue_text, + Publication.pub_url, + Publication.pdf_url, + ScholarPublication.is_read, + ScholarPublication.first_seen_run_id, + ScholarPublication.created_at, + ) + .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) + .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) + .limit(limit) + ) + if scholar_profile_id is not None: + stmt = stmt.where(ScholarProfile.id == scholar_profile_id) + if mode == MODE_UNREAD: + stmt = stmt.where(ScholarPublication.is_read.is_(False)) + if mode == MODE_LATEST: + if latest_run_id is None: + return stmt.where(False) + stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) + return stmt + + +def publication_list_item_from_row( + row: tuple, + *, + latest_run_id: int | None, +) -> PublicationListItem: + ( + publication_id, + scholar_profile_id, + display_name, + scholar_id, + title_raw, + year, + citation_count, + venue_text, + pub_url, + pdf_url, + is_read, + first_seen_run_id, + created_at, + ) = row + return PublicationListItem( + publication_id=int(publication_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + title=title_raw, + year=year, + citation_count=_normalized_citation_count(citation_count), + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + is_read=bool(is_read), + first_seen_at=created_at, + is_new_in_latest_run=( + latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id + ), + ) + + +def unread_item_from_row(row: tuple) -> UnreadPublicationItem: + ( + publication_id, + scholar_profile_id, + display_name, + scholar_id, + title_raw, + year, + citation_count, + venue_text, + pub_url, + pdf_url, + _is_read, + _first_seen_run_id, + _created_at, + ) = row + return UnreadPublicationItem( + publication_id=int(publication_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + title=title_raw, + year=year, + citation_count=_normalized_citation_count(citation_count), + venue_text=venue_text, + pub_url=pub_url, + pdf_url=pdf_url, + ) diff --git a/app/services/domains/publications/read_state.py b/app/services/domains/publications/read_state.py new file mode 100644 index 0000000..37a9815 --- /dev/null +++ b/app/services/domains/publications/read_state.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from sqlalchemy import select, tuple_, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ScholarProfile, ScholarPublication + + +def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[int, int]]: + pairs: set[tuple[int, int]] = set() + for scholar_profile_id, publication_id in selections: + normalized = (int(scholar_profile_id), int(publication_id)) + if normalized[0] <= 0 or normalized[1] <= 0: + continue + pairs.add(normalized) + return pairs + + +async def mark_all_unread_as_read_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> int: + scholar_ids = ( + select(ScholarProfile.id) + .where(ScholarProfile.user_id == user_id) + .scalar_subquery() + ) + stmt = ( + update(ScholarPublication) + .where( + ScholarPublication.scholar_profile_id.in_(scholar_ids), + ScholarPublication.is_read.is_(False), + ) + .values(is_read=True) + ) + result = await db_session.execute(stmt) + await db_session.commit() + return int(result.rowcount or 0) + + +async def mark_selected_as_read_for_user( + db_session: AsyncSession, + *, + user_id: int, + selections: list[tuple[int, int]], +) -> int: + normalized_pairs = _normalized_selection_pairs(selections) + if not normalized_pairs: + return 0 + + scholar_ids = ( + select(ScholarProfile.id) + .where(ScholarProfile.user_id == user_id) + .scalar_subquery() + ) + stmt = ( + update(ScholarPublication) + .where( + ScholarPublication.scholar_profile_id.in_(scholar_ids), + tuple_( + ScholarPublication.scholar_profile_id, + ScholarPublication.publication_id, + ).in_(list(normalized_pairs)), + ScholarPublication.is_read.is_(False), + ) + .values(is_read=True) + ) + result = await db_session.execute(stmt) + await db_session.commit() + return int(result.rowcount or 0) diff --git a/app/services/domains/publications/types.py b/app/services/domains/publications/types.py new file mode 100644 index 0000000..d64a812 --- /dev/null +++ b/app/services/domains/publications/types.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class PublicationListItem: + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + pdf_url: str | None + is_read: bool + first_seen_at: datetime + is_new_in_latest_run: bool + + +@dataclass(frozen=True) +class UnreadPublicationItem: + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + pdf_url: str | None diff --git a/app/services/domains/runs/__init__.py b/app/services/domains/runs/__init__.py new file mode 100644 index 0000000..848f6e0 --- /dev/null +++ b/app/services/domains/runs/__init__.py @@ -0,0 +1 @@ +from app.services.domains.runs.application import * diff --git a/app/services/domains/runs/application.py b/app/services/domains/runs/application.py new file mode 100644 index 0000000..0019446 --- /dev/null +++ b/app/services/domains/runs/application.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from app.services.domains.runs.queue_service import ( + clear_queue_item_for_user, + drop_queue_item_for_user, + get_queue_item_for_user, + list_queue_items_for_user, + queue_status_counts_for_user, + retry_queue_item_for_user, +) +from app.services.domains.runs.runs_service import ( + get_manual_run_by_idempotency_key, + get_run_for_user, + list_recent_runs_for_user, + list_runs_for_user, +) +from app.services.domains.runs.summary import extract_run_summary +from app.services.domains.runs.types import ( + QUEUE_STATUS_DROPPED, + QUEUE_STATUS_QUEUED, + QUEUE_STATUS_RETRYING, + QueueClearResult, + QueueListItem, + QueueTransitionError, +) + +__all__ = [ + "QUEUE_STATUS_QUEUED", + "QUEUE_STATUS_RETRYING", + "QUEUE_STATUS_DROPPED", + "QueueListItem", + "QueueClearResult", + "QueueTransitionError", + "extract_run_summary", + "list_recent_runs_for_user", + "list_runs_for_user", + "get_run_for_user", + "get_manual_run_by_idempotency_key", + "list_queue_items_for_user", + "get_queue_item_for_user", + "retry_queue_item_for_user", + "drop_queue_item_for_user", + "clear_queue_item_for_user", + "queue_status_counts_for_user", +] diff --git a/app/services/domains/runs/queue_queries.py b/app/services/domains/runs/queue_queries.py new file mode 100644 index 0000000..7d2e501 --- /dev/null +++ b/app/services/domains/runs/queue_queries.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from sqlalchemy import and_, select + +from app.db.models import IngestionQueueItem, ScholarProfile +from app.services.domains.runs.types import QueueListItem + + +def queue_item_columns() -> tuple: + return ( + IngestionQueueItem.id, + IngestionQueueItem.scholar_profile_id, + ScholarProfile.display_name, + ScholarProfile.scholar_id, + IngestionQueueItem.status, + IngestionQueueItem.reason, + IngestionQueueItem.dropped_reason, + IngestionQueueItem.attempt_count, + IngestionQueueItem.resume_cstart, + IngestionQueueItem.next_attempt_dt, + IngestionQueueItem.updated_at, + IngestionQueueItem.last_error, + IngestionQueueItem.last_run_id, + ) + + +def queue_item_select(*, user_id: int): + return ( + select(*queue_item_columns()) + .join( + ScholarProfile, + and_( + ScholarProfile.id == IngestionQueueItem.scholar_profile_id, + ScholarProfile.user_id == IngestionQueueItem.user_id, + ), + ) + .where(IngestionQueueItem.user_id == user_id) + ) + + +def queue_list_item_from_row(row: tuple) -> QueueListItem: + ( + item_id, + scholar_profile_id, + display_name, + scholar_id, + status, + reason, + dropped_reason, + attempt_count, + resume_cstart, + next_attempt_dt, + updated_at, + last_error, + last_run_id, + ) = row + return QueueListItem( + id=int(item_id), + scholar_profile_id=int(scholar_profile_id), + scholar_label=(display_name or scholar_id), + status=str(status), + reason=str(reason), + dropped_reason=dropped_reason, + attempt_count=int(attempt_count or 0), + resume_cstart=int(resume_cstart or 0), + next_attempt_dt=next_attempt_dt, + updated_at=updated_at, + last_error=last_error, + last_run_id=int(last_run_id) if last_run_id is not None else None, + ) diff --git a/app/services/domains/runs/queue_service.py b/app/services/domains/runs/queue_service.py new file mode 100644 index 0000000..d69ce21 --- /dev/null +++ b/app/services/domains/runs/queue_service.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +from sqlalchemy import case, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import IngestionQueueItem +from app.services.domains.ingestion import queue as queue_mutations +from app.services.domains.runs.queue_queries import queue_item_select, queue_list_item_from_row +from app.services.domains.runs.types import ( + QUEUE_STATUS_DROPPED, + QUEUE_STATUS_QUEUED, + QUEUE_STATUS_RETRYING, + QueueClearResult, + QueueListItem, + QueueTransitionError, +) + + +async def list_queue_items_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 200, +) -> list[QueueListItem]: + result = await db_session.execute( + queue_item_select(user_id=user_id) + .order_by( + case((IngestionQueueItem.status == QUEUE_STATUS_DROPPED, 1), else_=0).asc(), + IngestionQueueItem.next_attempt_dt.asc(), + IngestionQueueItem.id.asc(), + ) + .limit(limit) + ) + return [queue_list_item_from_row(row) for row in result.all()] + + +async def get_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + queue_item_select(user_id=user_id) + .where(IngestionQueueItem.id == queue_item_id) + .limit(1) + ) + row = result.one_or_none() + if row is None: + return None + return queue_list_item_from_row(row) + + +async def retry_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QUEUE_STATUS_QUEUED: + raise QueueTransitionError( + code="queue_item_already_queued", + message="Queue item is already queued.", + current_status=item.status, + ) + if item.status == QUEUE_STATUS_RETRYING: + raise QueueTransitionError( + code="queue_item_retrying", + message="Queue item is currently retrying.", + current_status=item.status, + ) + + await queue_mutations.mark_queued_now( + db_session, + job_id=item.id, + reason="manual_retry", + reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED), + ) + await db_session.commit() + return await get_queue_item_for_user( + db_session, + user_id=user_id, + queue_item_id=queue_item_id, + ) + + +async def drop_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueListItem | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status == QUEUE_STATUS_DROPPED: + raise QueueTransitionError( + code="queue_item_already_dropped", + message="Queue item is already dropped.", + current_status=item.status, + ) + + await queue_mutations.mark_dropped( + db_session, + job_id=int(item.id), + reason="manual_drop", + ) + await db_session.commit() + return await get_queue_item_for_user( + db_session, + user_id=user_id, + queue_item_id=queue_item_id, + ) + + +async def clear_queue_item_for_user( + db_session: AsyncSession, + *, + user_id: int, + queue_item_id: int, +) -> QueueClearResult | None: + result = await db_session.execute( + select(IngestionQueueItem).where( + IngestionQueueItem.id == queue_item_id, + IngestionQueueItem.user_id == user_id, + ) + ) + item = result.scalar_one_or_none() + if item is None: + return None + if item.status != QUEUE_STATUS_DROPPED: + raise QueueTransitionError( + code="queue_item_not_dropped", + message="Queue item can only be cleared after it is dropped.", + current_status=item.status, + ) + + item_id = int(item.id) + previous_status = str(item.status) + deleted = await queue_mutations.delete_job_by_id(db_session, job_id=item_id) + await db_session.commit() + if not deleted: + return None + return QueueClearResult(queue_item_id=item_id, previous_status=previous_status) + + +async def queue_status_counts_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> dict[str, int]: + result = await db_session.execute( + select( + IngestionQueueItem.status, + func.count(IngestionQueueItem.id), + ) + .where(IngestionQueueItem.user_id == user_id) + .group_by(IngestionQueueItem.status) + ) + counts: dict[str, int] = { + QUEUE_STATUS_QUEUED: 0, + QUEUE_STATUS_RETRYING: 0, + QUEUE_STATUS_DROPPED: 0, + } + for status, count in result.all(): + counts[str(status)] = int(count or 0) + return counts diff --git a/app/services/domains/runs/runs_service.py b/app/services/domains/runs/runs_service.py new file mode 100644 index 0000000..1cb9aa2 --- /dev/null +++ b/app/services/domains/runs/runs_service.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from sqlalchemy import or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import CrawlRun, RunStatus, RunTriggerType + + +async def list_recent_runs_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 20, +) -> list[CrawlRun]: + result = await db_session.execute( + select(CrawlRun) + .where(CrawlRun.user_id == user_id) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(limit) + ) + return list(result.scalars().all()) + + +async def list_runs_for_user( + db_session: AsyncSession, + *, + user_id: int, + limit: int = 100, + failed_only: bool = False, +) -> list[CrawlRun]: + stmt = ( + select(CrawlRun) + .where(CrawlRun.user_id == user_id) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(limit) + ) + if failed_only: + stmt = stmt.where( + CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE]) + ) + result = await db_session.execute(stmt) + return list(result.scalars().all()) + + +async def get_run_for_user( + db_session: AsyncSession, + *, + user_id: int, + run_id: int, +) -> CrawlRun | None: + result = await db_session.execute( + select(CrawlRun).where( + CrawlRun.user_id == user_id, + CrawlRun.id == run_id, + ) + ) + return result.scalar_one_or_none() + + +async def get_manual_run_by_idempotency_key( + db_session: AsyncSession, + *, + user_id: int, + idempotency_key: str, +) -> CrawlRun | None: + result = await db_session.execute( + select(CrawlRun) + .where( + CrawlRun.user_id == user_id, + CrawlRun.trigger_type == RunTriggerType.MANUAL, + or_( + CrawlRun.idempotency_key == idempotency_key, + CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key, + ), + ) + .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) + .limit(1) + ) + return result.scalar_one_or_none() diff --git a/app/services/domains/runs/summary.py b/app/services/domains/runs/summary.py new file mode 100644 index 0000000..9555558 --- /dev/null +++ b/app/services/domains/runs/summary.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from typing import Any + + +def _safe_int(value: object, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _summary_dict(error_log: object) -> dict[str, Any]: + if not isinstance(error_log, dict): + return {} + summary = error_log.get("summary") + if not isinstance(summary, dict): + return {} + return summary + + +def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]: + value = summary.get(key) + if not isinstance(value, dict): + return {} + return { + str(item_key): _safe_int(item_value, 0) + for item_key, item_value in value.items() + if isinstance(item_key, str) + } + + +def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]: + value = summary.get(key) + if not isinstance(value, dict): + return {} + return { + str(item_key): bool(item_value) + for item_key, item_value in value.items() + if isinstance(item_key, str) + } + + +def _retry_counts(summary: dict[str, Any]) -> dict[str, int]: + retry_counts = summary.get("retry_counts") + if not isinstance(retry_counts, dict): + retry_counts = {} + return { + "retries_scheduled_count": _safe_int( + retry_counts.get("retries_scheduled_count", 0), + 0, + ), + "scholars_with_retries_count": _safe_int( + retry_counts.get("scholars_with_retries_count", 0), + 0, + ), + "retry_exhausted_count": _safe_int( + retry_counts.get("retry_exhausted_count", 0), + 0, + ), + } + + +def extract_run_summary(error_log: object) -> dict[str, Any]: + summary = _summary_dict(error_log) + return { + "succeeded_count": _safe_int(summary.get("succeeded_count", 0)), + "failed_count": _safe_int(summary.get("failed_count", 0)), + "partial_count": _safe_int(summary.get("partial_count", 0)), + "failed_state_counts": _summary_int_dict(summary, "failed_state_counts"), + "failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"), + "scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"), + "retry_counts": _retry_counts(summary), + "alert_thresholds": _summary_int_dict(summary, "alert_thresholds"), + "alert_flags": _summary_bool_dict(summary, "alert_flags"), + } diff --git a/app/services/domains/runs/types.py b/app/services/domains/runs/types.py new file mode 100644 index 0000000..2524269 --- /dev/null +++ b/app/services/domains/runs/types.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +QUEUE_STATUS_QUEUED = "queued" +QUEUE_STATUS_RETRYING = "retrying" +QUEUE_STATUS_DROPPED = "dropped" + + +@dataclass(frozen=True) +class QueueListItem: + id: int + scholar_profile_id: int + scholar_label: str + status: str + reason: str + dropped_reason: str | None + attempt_count: int + resume_cstart: int + next_attempt_dt: datetime | None + updated_at: datetime + last_error: str | None + last_run_id: int | None + + +@dataclass(frozen=True) +class QueueClearResult: + queue_item_id: int + previous_status: str + + +class QueueTransitionError(RuntimeError): + def __init__(self, *, code: str, message: str, current_status: str) -> None: + super().__init__(message) + self.code = code + self.message = message + self.current_status = current_status diff --git a/app/services/domains/scholar/__init__.py b/app/services/domains/scholar/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/app/services/domains/scholar/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/app/services/domains/scholar/author_rows.py b/app/services/domains/scholar/author_rows.py new file mode 100644 index 0000000..8acd783 --- /dev/null +++ b/app/services/domains/scholar/author_rows.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import re +from html.parser import HTMLParser +from typing import Any +from urllib.parse import parse_qs, urlparse + +from app.services.domains.scholar.parser_constants import AUTHOR_SEARCH_MARKER_KEYS +from app.services.domains.scholar.parser_types import ScholarSearchCandidate +from app.services.domains.scholar.parser_utils import ( + attr_class, + attr_href, + attr_src, + build_absolute_scholar_url, + normalize_space, +) + + +def parse_scholar_id_from_href(href: str | None) -> str | None: + if not href: + return None + parsed = urlparse(href) + query = parse_qs(parsed.query) + user_values = query.get("user") + if not user_values: + return None + candidate = user_values[0].strip() + return candidate or None + + +def _extract_verified_email_domain(value: str | None) -> str | None: + if not value: + return None + match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I) + if not match: + return None + domain = normalize_space(match.group(1)) + return domain or None + + +class ScholarAuthorSearchParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.candidates: list[ScholarSearchCandidate] = [] + self._candidate: dict[str, Any] | None = None + + def _begin_candidate(self) -> None: + self._candidate = { + "depth": 1, + "name_href": None, + "name_parts": [], + "aff_depth": 0, + "aff_parts": [], + "name_depth": 0, + "eml_depth": 0, + "eml_parts": [], + "cby_depth": 0, + "cby_parts": [], + "interest_depth": 0, + "interest_parts": [], + "interests": [], + "image_src": None, + } + + def _increment_capture_depths(self) -> None: + if self._candidate is None: + return + for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"): + if self._candidate[key] > 0: + self._candidate[key] += 1 + + def _finalize_candidate(self) -> None: + if self._candidate is None: + return + + name = normalize_space("".join(self._candidate["name_parts"])) + scholar_id = parse_scholar_id_from_href(self._candidate["name_href"]) + if not name or not scholar_id: + return + + affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None + email_domain = _extract_verified_email_domain( + normalize_space("".join(self._candidate["eml_parts"])) or None + ) + cited_by_text = normalize_space("".join(self._candidate["cby_parts"])) + cited_by_match = re.search(r"\d+", cited_by_text) + cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None + + seen_interests: set[str] = set() + interests: list[str] = [] + for interest in self._candidate["interests"]: + normalized = normalize_space(interest) + if not normalized or normalized in seen_interests: + continue + seen_interests.add(normalized) + interests.append(normalized) + + profile_url = build_absolute_scholar_url(self._candidate["name_href"]) + if not profile_url: + profile_url = ( + "https://scholar.google.com/citations" + f"?hl=en&user={scholar_id}" + ) + + self.candidates.append( + ScholarSearchCandidate( + scholar_id=scholar_id, + display_name=name, + affiliation=affiliation, + email_domain=email_domain, + cited_by_count=cited_by_count, + interests=interests, + profile_url=profile_url, + profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]), + ) + ) + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + classes = attr_class(attrs) + + if self._candidate is None: + if tag == "div" and "gsc_1usr" in classes: + self._begin_candidate() + return + + self._candidate["depth"] += 1 + self._increment_capture_depths() + + if tag == "a" and "gs_ai_name" in classes: + self._candidate["name_depth"] = 1 + self._candidate["name_href"] = attr_href(attrs) + return + + if tag == "div" and "gs_ai_aff" in classes: + self._candidate["aff_depth"] = 1 + return + + if tag == "div" and "gs_ai_eml" in classes: + self._candidate["eml_depth"] = 1 + return + + if tag == "div" and "gs_ai_cby" in classes: + self._candidate["cby_depth"] = 1 + return + + if tag == "a" and "gs_ai_one_int" in classes: + self._candidate["interest_depth"] = 1 + self._candidate["interest_parts"] = [] + return + + if tag == "img" and self._candidate["image_src"] is None: + self._candidate["image_src"] = attr_src(attrs) + + def handle_data(self, data: str) -> None: + if self._candidate is None: + return + if self._candidate["name_depth"] > 0: + self._candidate["name_parts"].append(data) + if self._candidate["aff_depth"] > 0: + self._candidate["aff_parts"].append(data) + if self._candidate["eml_depth"] > 0: + self._candidate["eml_parts"].append(data) + if self._candidate["cby_depth"] > 0: + self._candidate["cby_parts"].append(data) + if self._candidate["interest_depth"] > 0: + self._candidate["interest_parts"].append(data) + + def _decrement_capture_depth(self, key: str) -> bool: + if self._candidate is None: + return False + if self._candidate[key] <= 0: + return False + self._candidate[key] -= 1 + return self._candidate[key] == 0 + + def handle_endtag(self, _tag: str) -> None: + if self._candidate is None: + return + + interest_closed = self._decrement_capture_depth("interest_depth") + self._decrement_capture_depth("name_depth") + self._decrement_capture_depth("aff_depth") + self._decrement_capture_depth("eml_depth") + self._decrement_capture_depth("cby_depth") + + if interest_closed: + interest_text = normalize_space("".join(self._candidate["interest_parts"])) + if interest_text: + self._candidate["interests"].append(interest_text) + self._candidate["interest_parts"] = [] + + self._candidate["depth"] -= 1 + if self._candidate["depth"] > 0: + return + + self._finalize_candidate() + self._candidate = None + + +def count_author_search_markers(html: str) -> dict[str, int]: + lowered = html.lower() + return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS} diff --git a/app/services/domains/scholar/parser.py b/app/services/domains/scholar/parser.py new file mode 100644 index 0000000..49dc773 --- /dev/null +++ b/app/services/domains/scholar/parser.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from app.services.domains.scholar.author_rows import ( + ScholarAuthorSearchParser, + count_author_search_markers, + parse_scholar_id_from_href, +) +from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE +from app.services.domains.scholar.parser_types import ( + ParseState, + ParsedAuthorSearchPage, + ParsedProfilePage, + PublicationCandidate, + ScholarSearchCandidate, +) +from app.services.domains.scholar.parser_utils import ( + attr_class, + attr_href, + attr_src, + build_absolute_scholar_url, + normalize_space, + strip_tags, +) +from app.services.domains.scholar.profile_rows import ( + ScholarRowParser, + count_markers, + extract_articles_range, + extract_profile_image_url, + extract_profile_name, + extract_rows, + has_operation_error_banner, + has_show_more_button, + parse_citation_count, + parse_cluster_id_from_href, + parse_publications, + parse_year, +) +from app.services.domains.scholar.source import FetchResult +from app.services.domains.scholar.state_detection import ( + classify_block_or_captcha_reason, + classify_network_error_reason, + detect_author_search_state, + detect_state, +) + + +def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: + publications, warnings = parse_publications(fetch_result.body) + marker_counts = count_markers(fetch_result.body) + visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() + + show_more = has_show_more_button(fetch_result.body) + operation_error_banner = has_operation_error_banner(fetch_result.body) + articles_range = extract_articles_range(fetch_result.body) + + if show_more: + warnings.append("possible_partial_page_show_more_present") + if operation_error_banner: + warnings.append("operation_error_banner_present") + + warnings = sorted(set(warnings)) + + state, state_reason = detect_state( + fetch_result, + publications, + marker_counts, + warnings=warnings, + has_show_more_button_flag=show_more, + articles_range=articles_range, + visible_text=visible_text, + ) + + return ParsedProfilePage( + state=state, + state_reason=state_reason, + profile_name=extract_profile_name(fetch_result.body), + profile_image_url=extract_profile_image_url(fetch_result.body), + publications=publications, + marker_counts=marker_counts, + warnings=warnings, + has_show_more_button=show_more, + has_operation_error_banner=operation_error_banner, + articles_range=articles_range, + ) + + +def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage: + parser = ScholarAuthorSearchParser() + parser.feed(fetch_result.body) + + marker_counts = count_author_search_markers(fetch_result.body) + visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() + warnings: list[str] = [] + if not parser.candidates: + warnings.append("no_author_candidates_detected") + + state, state_reason = detect_author_search_state( + fetch_result, + parser.candidates, + marker_counts, + visible_text=visible_text, + ) + + return ParsedAuthorSearchPage( + state=state, + state_reason=state_reason, + candidates=parser.candidates, + marker_counts=marker_counts, + warnings=warnings, + ) diff --git a/app/services/domains/scholar/parser_constants.py b/app/services/domains/scholar/parser_constants.py new file mode 100644 index 0000000..c3c7a15 --- /dev/null +++ b/app/services/domains/scholar/parser_constants.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import re + +BLOCKED_KEYWORDS = [ + "unusual traffic", + "sorry/index", + "not a robot", + "our systems have detected", + "automated queries", + "recaptcha", + "captcha", +] + +NO_RESULTS_KEYWORDS = [ + "didn't match any articles", + "did not match any articles", + "no articles", + "no documents", +] + +NO_AUTHOR_RESULTS_KEYWORDS = [ + "didn't match any user profiles", + "did not match any user profiles", + "didn't match any scholars", + "did not match any scholars", + "no user profiles", +] + +MARKER_KEYS = [ + "gsc_a_tr", + "gsc_a_at", + "gsc_a_ac", + "gsc_a_h", + "gsc_a_y", + "gs_gray", + "gsc_prf_in", + "gsc_rsb_st", +] + +AUTHOR_SEARCH_MARKER_KEYS = [ + "gsc_1usr", + "gs_ai_name", + "gs_ai_aff", + "gs_ai_eml", + "gs_ai_cby", + "gs_ai_one_int", +] + +NETWORK_DNS_ERROR_KEYWORDS = [ + "temporary failure in name resolution", + "name or service not known", + "nodename nor servname provided", + "getaddrinfo failed", +] + +NETWORK_TIMEOUT_KEYWORDS = [ + "timed out", + "timeout", +] + +NETWORK_TLS_ERROR_KEYWORDS = [ + "ssl", + "tls", + "certificate verify failed", +] + +TAG_RE = re.compile(r"<[^>]+>", re.S) +SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?", re.I | re.S) +SHOW_MORE_BUTTON_RE = re.compile( + r"]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>", + re.I | re.S, +) + +PROFILE_ROW_PARSER_DIRECT_MARKERS = ( + "gs_ggs", + "gs_ggsd", + "gs_ggsa", + "gs_or_ggsm", +) + +PROFILE_ROW_DIRECT_LABEL_TOKENS = ( + "pdf", + "[pdf]", + "full text", + "download", +) diff --git a/app/services/domains/scholar/parser_types.py b/app/services/domains/scholar/parser_types.py new file mode 100644 index 0000000..a536389 --- /dev/null +++ b/app/services/domains/scholar/parser_types.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + + +class ParseState(StrEnum): + OK = "ok" + NO_RESULTS = "no_results" + BLOCKED_OR_CAPTCHA = "blocked_or_captcha" + LAYOUT_CHANGED = "layout_changed" + NETWORK_ERROR = "network_error" + + +@dataclass(frozen=True) +class PublicationCandidate: + title: str + title_url: str | None + cluster_id: str | None + year: int | None + citation_count: int | None + authors_text: str | None + venue_text: str | None + pdf_url: str | None + + +@dataclass(frozen=True) +class ScholarSearchCandidate: + scholar_id: str + display_name: str + affiliation: str | None + email_domain: str | None + cited_by_count: int | None + interests: list[str] + profile_url: str + profile_image_url: str | None + + +@dataclass(frozen=True) +class ParsedProfilePage: + state: ParseState + state_reason: str + profile_name: str | None + profile_image_url: str | None + publications: list[PublicationCandidate] + marker_counts: dict[str, int] + warnings: list[str] + has_show_more_button: bool + has_operation_error_banner: bool + articles_range: str | None + + +@dataclass(frozen=True) +class ParsedAuthorSearchPage: + state: ParseState + state_reason: str + candidates: list[ScholarSearchCandidate] + marker_counts: dict[str, int] + warnings: list[str] diff --git a/app/services/domains/scholar/parser_utils.py b/app/services/domains/scholar/parser_utils.py new file mode 100644 index 0000000..1bf8687 --- /dev/null +++ b/app/services/domains/scholar/parser_utils.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from html import unescape + +from app.services.domains.scholar.parser_constants import TAG_RE + + +def normalize_space(value: str) -> str: + return " ".join(unescape(value).split()) + + +def strip_tags(value: str) -> str: + return normalize_space(TAG_RE.sub(" ", value)) + + +def attr_class(attrs: list[tuple[str, str | None]]) -> str: + for name, raw_value in attrs: + if name.lower() == "class": + return raw_value or "" + return "" + + +def attr_href(attrs: list[tuple[str, str | None]]) -> str | None: + for name, raw_value in attrs: + if name.lower() == "href": + return raw_value + return None + + +def attr_src(attrs: list[tuple[str, str | None]]) -> str | None: + for name, raw_value in attrs: + if name.lower() == "src": + return raw_value + return None + + +def build_absolute_scholar_url(path_or_url: str | None) -> str | None: + if not path_or_url: + return None + from urllib.parse import urljoin + + return urljoin("https://scholar.google.com", path_or_url) diff --git a/app/services/domains/scholar/profile_rows.py b/app/services/domains/scholar/profile_rows.py new file mode 100644 index 0000000..896bcaa --- /dev/null +++ b/app/services/domains/scholar/profile_rows.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +import re +from html.parser import HTMLParser +from typing import Any +from urllib.parse import parse_qs, urlparse + +from app.services.domains.scholar.parser_constants import ( + MARKER_KEYS, + PROFILE_ROW_DIRECT_LABEL_TOKENS, + PROFILE_ROW_PARSER_DIRECT_MARKERS, + SHOW_MORE_BUTTON_RE, +) +from app.services.domains.scholar.parser_types import PublicationCandidate +from app.services.domains.scholar.parser_utils import ( + attr_class, + attr_href, + attr_src, + build_absolute_scholar_url, + normalize_space, + strip_tags, +) + + +class ScholarRowParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.title_href: str | None = None + self.direct_download_href: str | None = None + self.title_parts: list[str] = [] + self.citation_parts: list[str] = [] + self.year_parts: list[str] = [] + self.gray_texts: list[str] = [] + + self._title_depth = 0 + self._citation_depth = 0 + self._year_depth = 0 + self._gray_stack: list[dict[str, Any]] = [] + self._direct_marker_depth = 0 + self._aux_link_stack: list[dict[str, Any]] = [] + + @staticmethod + def _contains_direct_marker(classes: str) -> bool: + lowered = classes.lower() + return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS) + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if self._title_depth > 0: + self._title_depth += 1 + if self._citation_depth > 0: + self._citation_depth += 1 + if self._year_depth > 0: + self._year_depth += 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] += 1 + if self._direct_marker_depth > 0: + self._direct_marker_depth += 1 + if self._aux_link_stack: + self._aux_link_stack[-1]["depth"] += 1 + + classes = attr_class(attrs) + if tag in {"div", "span"} and self._contains_direct_marker(classes): + self._direct_marker_depth = 1 + + if tag == "a" and "gsc_a_at" in classes: + self._title_depth = 1 + self.title_href = attr_href(attrs) + return + + if tag == "a" and "gsc_a_ac" in classes: + self._citation_depth = 1 + return + + if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes): + self._year_depth = 1 + return + + if tag == "div" and "gs_gray" in classes: + self._gray_stack.append({"depth": 1, "parts": []}) + return + + if tag == "a": + href = attr_href(attrs) + self._aux_link_stack.append( + { + "depth": 1, + "classes": classes, + "href": href, + "parts": [], + } + ) + + def handle_data(self, data: str) -> None: + if self._title_depth > 0: + self.title_parts.append(data) + if self._citation_depth > 0: + self.citation_parts.append(data) + if self._year_depth > 0: + self.year_parts.append(data) + if self._gray_stack: + self._gray_stack[-1]["parts"].append(data) + if self._aux_link_stack: + self._aux_link_stack[-1]["parts"].append(data) + + def _capture_direct_download_href(self, link_info: dict[str, Any]) -> None: + if self.direct_download_href: + return + href = link_info.get("href") + if not href: + return + classes = str(link_info.get("classes") or "") + label = normalize_space("".join(link_info.get("parts") or [])).lower() + if "gs_ggsd" in classes or "gs_ggs" in classes or "gs_ggsa" in classes: + self.direct_download_href = href + return + label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS) + if self._direct_marker_depth > 0 and label_match: + self.direct_download_href = href + + def handle_endtag(self, tag: str) -> None: + if self._title_depth > 0: + self._title_depth -= 1 + if self._citation_depth > 0: + self._citation_depth -= 1 + if self._year_depth > 0: + self._year_depth -= 1 + if self._gray_stack: + self._gray_stack[-1]["depth"] -= 1 + if self._gray_stack[-1]["depth"] == 0: + text_value = normalize_space("".join(self._gray_stack[-1]["parts"])) + if text_value: + self.gray_texts.append(text_value) + self._gray_stack.pop() + if self._aux_link_stack: + self._aux_link_stack[-1]["depth"] -= 1 + if self._aux_link_stack[-1]["depth"] == 0: + link_info = self._aux_link_stack.pop() + self._capture_direct_download_href(link_info) + if self._direct_marker_depth > 0: + self._direct_marker_depth -= 1 + + +def extract_rows(html: str) -> list[str]: + pattern = re.compile( + r"]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)", + re.I | re.S, + ) + return [match.group(1) for match in pattern.finditer(html)] + + +def parse_cluster_id_from_href(href: str | None) -> str | None: + if not href: + return None + parsed = urlparse(href) + query = parse_qs(parsed.query) + + citation_for_view = query.get("citation_for_view") + if citation_for_view: + token = citation_for_view[0].strip() + if token: + if ":" in token: + return token.rsplit(":", 1)[-1] or None + return token + + cluster = query.get("cluster") + if cluster: + token = cluster[0].strip() + if token: + return token + return None + + +def parse_year(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + match = re.search(r"\b(19|20)\d{2}\b", text) + if not match: + return None + try: + return int(match.group(0)) + except ValueError: + return None + + +def parse_citation_count(parts: list[str]) -> int | None: + text = normalize_space(" ".join(parts)) + if not text: + return 0 + match = re.search(r"\d+", text) + if not match: + return None + return int(match.group(0)) + + +def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]: + parser = ScholarRowParser() + parser.feed(row_html) + warnings: list[str] = [] + title = normalize_space("".join(parser.title_parts)) + if not title: + warnings.append("row_missing_title") + return None, warnings + if not parser.title_href: + warnings.append("row_missing_title_href") + + citation_text = normalize_space(" ".join(parser.citation_parts)) + citation_count = parse_citation_count(parser.citation_parts) + if citation_text and citation_count is None: + warnings.append("layout_row_citation_unparseable") + + year_text = normalize_space(" ".join(parser.year_parts)) + year = parse_year(parser.year_parts) + if year_text and year is None: + warnings.append("layout_row_year_unparseable") + + authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None + venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None + return ( + PublicationCandidate( + title=title, + title_url=parser.title_href, + cluster_id=parse_cluster_id_from_href(parser.title_href), + year=year, + citation_count=citation_count, + authors_text=authors_text, + venue_text=venue_text, + pdf_url=build_absolute_scholar_url(parser.direct_download_href), + ), + warnings, + ) + + +def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]: + rows = extract_rows(html) + warnings: list[str] = [] + publications: list[PublicationCandidate] = [] + + for row_html in rows: + publication, row_warnings = _parse_publication_row(row_html) + warnings.extend(row_warnings) + if publication is None: + continue + publications.append(publication) + + if not rows: + warnings.append("no_rows_detected") + if rows and not publications: + warnings.append("layout_all_rows_unparseable") + + return publications, sorted(set(warnings)) + + +def extract_profile_name(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def extract_profile_image_url(html: str) -> str | None: + og_image_pattern = re.compile( + r"]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>", + re.I | re.S, + ) + og_match = og_image_pattern.search(html) + if og_match: + value = normalize_space(og_match.group(1)) + absolute = build_absolute_scholar_url(value) + if absolute: + return absolute + + image_pattern = re.compile( + r"]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>", + re.I | re.S, + ) + image_match = image_pattern.search(html) + if not image_match: + return None + + value = normalize_space(image_match.group(1)) + return build_absolute_scholar_url(value) + + +def extract_articles_range(html: str) -> str | None: + pattern = re.compile( + r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)]+>", + re.I | re.S, + ) + match = pattern.search(html) + if not match: + return None + value = strip_tags(match.group(1)) + return value or None + + +def has_show_more_button(html: str) -> bool: + match = SHOW_MORE_BUTTON_RE.search(html) + if match is None: + return False + + button_tag = match.group(0).lower() + if "disabled" in button_tag: + return False + if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag: + return False + if "gs_dis" in button_tag: + return False + return True + + +def has_operation_error_banner(html: str) -> bool: + lowered = html.lower() + if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered: + return False + return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered + + +def count_markers(html: str) -> dict[str, int]: + lowered = html.lower() + return {key: lowered.count(key.lower()) for key in MARKER_KEYS} diff --git a/app/services/domains/scholar/source.py b/app/services/domains/scholar/source.py new file mode 100644 index 0000000..83da165 --- /dev/null +++ b/app/services/domains/scholar/source.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import asyncio +import logging +import random +from dataclasses import dataclass +from typing import Protocol +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations" +DEFAULT_PAGE_SIZE = 100 + +DEFAULT_USER_AGENTS = [ + ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + ), + ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 " + "(KHTML, like Gecko) Version/18.1 Safari/605.1.15" + ), + ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " + "Gecko/20100101 Firefox/131.0" + ), +] + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FetchResult: + requested_url: str + status_code: int | None + final_url: str | None + body: str + error: str | None + + +class ScholarSource(Protocol): + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + ... + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int, + ) -> FetchResult: + ... + + async def fetch_author_search_html( + self, + query: str, + *, + start: int, + ) -> FetchResult: + ... + + +class LiveScholarSource: + def __init__( + self, + *, + timeout_seconds: float = 25.0, + user_agents: list[str] | None = None, + ) -> None: + self._timeout_seconds = timeout_seconds + self._user_agents = user_agents or DEFAULT_USER_AGENTS + + async def fetch_profile_html(self, scholar_id: str) -> FetchResult: + return await self.fetch_profile_page_html( + scholar_id, + cstart=0, + pagesize=DEFAULT_PAGE_SIZE, + ) + + async def fetch_profile_page_html( + self, + scholar_id: str, + *, + cstart: int, + pagesize: int = DEFAULT_PAGE_SIZE, + ) -> FetchResult: + requested_url = _build_profile_url( + scholar_id=scholar_id, + cstart=cstart, + pagesize=pagesize, + ) + logger.debug( + "scholar_source.fetch_started", + extra={ + "event": "scholar_source.fetch_started", + "scholar_id": scholar_id, + "requested_url": requested_url, + "cstart": cstart, + "pagesize": pagesize, + }, + ) + return await asyncio.to_thread(self._fetch_sync, requested_url) + + async def fetch_author_search_html( + self, + query: str, + *, + start: int = 0, + ) -> FetchResult: + requested_url = _build_author_search_url( + query=query, + start=start, + ) + logger.debug( + "scholar_source.search_fetch_started", + extra={ + "event": "scholar_source.search_fetch_started", + "query": query, + "requested_url": requested_url, + "start": start, + }, + ) + return await asyncio.to_thread(self._fetch_sync, requested_url) + + def _build_request(self, requested_url: str) -> Request: + return Request( + requested_url, + headers={ + "User-Agent": random.choice(self._user_agents), + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", + "Connection": "close", + }, + ) + + @staticmethod + def _http_error_body(exc: HTTPError) -> str: + try: + return exc.read().decode("utf-8", errors="replace") + except Exception: + return "" + + @staticmethod + def _network_error_result(requested_url: str, exc: URLError) -> FetchResult: + logger.warning( + "scholar_source.fetch_network_error", + extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url}, + ) + return FetchResult( + requested_url=requested_url, + status_code=None, + final_url=None, + body="", + error=str(exc), + ) + + @staticmethod + def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult: + logger.warning( + "scholar_source.fetch_http_error", + extra={ + "event": "scholar_source.fetch_http_error", + "requested_url": requested_url, + "status_code": exc.code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=exc.code, + final_url=exc.geturl(), + body=LiveScholarSource._http_error_body(exc), + error=str(exc), + ) + + @staticmethod + def _success_result(requested_url: str, response) -> FetchResult: + body = response.read().decode("utf-8", errors="replace") + status_code = getattr(response, "status", 200) + logger.debug( + "scholar_source.fetch_succeeded", + extra={ + "event": "scholar_source.fetch_succeeded", + "requested_url": requested_url, + "status_code": status_code, + }, + ) + return FetchResult( + requested_url=requested_url, + status_code=status_code, + final_url=response.geturl(), + body=body, + error=None, + ) + + def _fetch_sync(self, requested_url: str) -> FetchResult: + request = self._build_request(requested_url) + + try: + with urlopen(request, timeout=self._timeout_seconds) as response: + return self._success_result(requested_url, response) + except HTTPError as exc: + return self._http_error_result(requested_url, exc) + except URLError as exc: + return self._network_error_result(requested_url, exc) + + +def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str: + query: dict[str, int | str] = {"hl": "en", "user": scholar_id} + if cstart > 0: + query["cstart"] = int(cstart) + if pagesize > 0: + query["pagesize"] = int(pagesize) + return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}" + + +def _build_author_search_url(*, query: str, start: int) -> str: + params: dict[str, int | str] = { + "hl": "en", + "view_op": "search_authors", + "mauthors": query, + } + if start > 0: + params["astart"] = int(start) + return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}" diff --git a/app/services/domains/scholar/state_detection.py b/app/services/domains/scholar/state_detection.py new file mode 100644 index 0000000..cdc8780 --- /dev/null +++ b/app/services/domains/scholar/state_detection.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from app.services.domains.scholar.parser_constants import ( + BLOCKED_KEYWORDS, + NETWORK_DNS_ERROR_KEYWORDS, + NETWORK_TIMEOUT_KEYWORDS, + NETWORK_TLS_ERROR_KEYWORDS, + NO_AUTHOR_RESULTS_KEYWORDS, + NO_RESULTS_KEYWORDS, +) +from app.services.domains.scholar.parser_types import ParseState, PublicationCandidate, ScholarSearchCandidate +from app.services.domains.scholar.source import FetchResult + + +def classify_network_error_reason(fetch_error: str | None) -> str: + lowered = (fetch_error or "").lower() + if lowered: + if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS): + return "network_dns_resolution_failed" + if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS): + return "network_timeout" + if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS): + return "network_tls_error" + if "connection reset" in lowered: + return "network_connection_reset" + if "connection refused" in lowered: + return "network_connection_refused" + if "network is unreachable" in lowered: + return "network_unreachable" + return "network_error_missing_status_code" + + +def classify_block_or_captcha_reason( + *, + status_code: int, + final_url: str, + body_lowered: str, +) -> str | None: + if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url): + return "blocked_accounts_redirect" + if status_code == 429: + return "blocked_http_429_rate_limited" + if status_code == 403: + if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url: + return "blocked_http_403_captcha_challenge" + return "blocked_http_403_forbidden" + if "sorry/index" in final_url or "sorry/index" in body_lowered: + return "blocked_google_sorry_challenge" + if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered: + return "blocked_unusual_traffic_detected" + if "automated queries" in body_lowered: + return "blocked_automated_queries_detected" + if "not a robot" in body_lowered: + return "blocked_not_a_robot_challenge" + if "recaptcha" in body_lowered: + return "blocked_recaptcha_challenge" + if "captcha" in body_lowered: + return "blocked_captcha_challenge" + if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS): + return "blocked_keyword_detected" + return None + + +def _warnings_contain(warnings: list[str], code: str) -> bool: + return any(item == code for item in warnings) + + +def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool: + if _warnings_contain(warnings, "layout_all_rows_unparseable"): + return True + if marker_counts.get("gsc_a_tr", 0) <= 0: + return False + if _warnings_contain(warnings, "row_missing_title"): + return True + return marker_counts.get("gsc_a_at", 0) <= 0 + + +def _first_layout_warning(warnings: list[str]) -> str | None: + for warning in warnings: + if warning.startswith("layout_"): + return warning + return None + + +def detect_state( + fetch_result: FetchResult, + publications: list[PublicationCandidate], + marker_counts: dict[str, int], + *, + warnings: list[str], + has_show_more_button_flag: bool, + articles_range: str | None, + visible_text: str, +) -> tuple[ParseState, str]: + if fetch_result.status_code is None: + return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) + + lowered = fetch_result.body.lower() + final = (fetch_result.final_url or "").lower() + status_code = int(fetch_result.status_code) + + block_reason = classify_block_or_captcha_reason( + status_code=status_code, + final_url=final, + body_lowered=lowered, + ) + if block_reason is not None: + return ParseState.BLOCKED_OR_CAPTCHA, block_reason + + if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS): + return ParseState.NO_RESULTS, "no_results_keyword_detected" + + layout_warning = _first_layout_warning(warnings) + if layout_warning is not None: + return ParseState.LAYOUT_CHANGED, layout_warning + + if _has_layout_row_failure(marker_counts, warnings): + return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable" + + if has_show_more_button_flag and not articles_range: + return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range" + + if not publications: + has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 + has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 + if not has_profile_markers and not has_table_markers: + return ParseState.LAYOUT_CHANGED, "layout_markers_missing" + return ParseState.OK, "no_rows_with_known_markers" + + return ParseState.OK, "publications_extracted" + + +def detect_author_search_state( + fetch_result: FetchResult, + candidates: list[ScholarSearchCandidate], + marker_counts: dict[str, int], + *, + visible_text: str, +) -> tuple[ParseState, str]: + if fetch_result.status_code is None: + return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) + + lowered = fetch_result.body.lower() + final = (fetch_result.final_url or "").lower() + status_code = int(fetch_result.status_code) + + block_reason = classify_block_or_captcha_reason( + status_code=status_code, + final_url=final, + body_lowered=lowered, + ) + if block_reason is not None: + return ParseState.BLOCKED_OR_CAPTCHA, block_reason + + if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS): + return ParseState.NO_RESULTS, "no_results_keyword_detected" + + if not candidates: + has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0 + if not has_search_markers: + return ParseState.NO_RESULTS, "no_search_candidates_detected" + return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable" + + return ParseState.OK, "author_candidates_extracted" diff --git a/app/services/domains/scholars/__init__.py b/app/services/domains/scholars/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/app/services/domains/scholars/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/app/services/domains/scholars/application.py b/app/services/domains/scholars/application.py new file mode 100644 index 0000000..65a3bee --- /dev/null +++ b/app/services/domains/scholars/application.py @@ -0,0 +1,966 @@ +from __future__ import annotations + +import asyncio +from dataclasses import replace +from datetime import datetime +from datetime import timedelta +from datetime import timezone +import logging +import os +import random +from uuid import uuid4 + +from sqlalchemy import delete, func, select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile +from app.services.domains.scholar.parser import ( + ParseState, + ParsedAuthorSearchPage, + parse_author_search_page, + parse_profile_page, +) +from app.services.domains.scholar.source import ScholarSource +from app.services.domains.scholars.constants import ( + ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES, + AUTHOR_SEARCH_LOCK_KEY, + AUTHOR_SEARCH_LOCK_NAMESPACE, + AUTHOR_SEARCH_RUNTIME_STATE_KEY, + DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, + DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, + DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, + DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD, + DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, + DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, + MAX_AUTHOR_SEARCH_LIMIT, + DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, + DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, + SEARCH_CACHED_BLOCK_REASON, + SEARCH_COOLDOWN_REASON, + SEARCH_DISABLED_REASON, +) +from app.services.domains.scholars.exceptions import ScholarServiceError +from app.services.domains.scholars.search_hints import ( + _merge_warnings, + _policy_blocked_author_search_result, + _trim_author_search_result, + resolve_profile_image, + scrape_state_hint, +) +from app.services.domains.scholars.uploads import ( + _ensure_upload_root, + _resolve_upload_path, + _safe_remove_upload, + resolve_upload_file_path, +) +from app.services.domains.scholars.validators import ( + normalize_display_name, + normalize_profile_image_url, + validate_scholar_id, +) +logger = logging.getLogger(__name__) + + +async def _acquire_author_search_lock(db_session: AsyncSession) -> None: + await db_session.execute( + text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"), + { + "namespace": AUTHOR_SEARCH_LOCK_NAMESPACE, + "lock_key": AUTHOR_SEARCH_LOCK_KEY, + }, + ) + + +async def _load_runtime_state_for_update( + db_session: AsyncSession, +) -> AuthorSearchRuntimeState: + result = await db_session.execute( + select(AuthorSearchRuntimeState) + .where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY) + .with_for_update() + ) + state = result.scalar_one_or_none() + if state is not None: + return state + + state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY) + db_session.add(state) + await db_session.flush() + return state + + +def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict: + return { + "state": parsed.state.value, + "state_reason": parsed.state_reason, + "marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()}, + "warnings": [str(value) for value in parsed.warnings], + "candidates": [ + { + "scholar_id": candidate.scholar_id, + "display_name": candidate.display_name, + "affiliation": candidate.affiliation, + "email_domain": candidate.email_domain, + "cited_by_count": candidate.cited_by_count, + "interests": [str(interest) for interest in candidate.interests], + "profile_url": candidate.profile_url, + "profile_image_url": candidate.profile_image_url, + } + for candidate in parsed.candidates + ], + } + + +def _payload_state(payload: dict[str, object]) -> ParseState | None: + state_raw = str(payload.get("state", "")).strip() + try: + return ParseState(state_raw) + except ValueError: + return None + + +def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]: + marker_counts_payload = payload.get("marker_counts") + if not isinstance(marker_counts_payload, dict): + return {} + parsed: dict[str, int] = {} + for key, value in marker_counts_payload.items(): + try: + parsed[str(key)] = int(value) + except (TypeError, ValueError): + continue + return parsed + + +def _payload_warnings(payload: dict[str, object]) -> list[str]: + warnings_payload = payload.get("warnings") + if not isinstance(warnings_payload, list): + return [] + return [str(value) for value in warnings_payload if isinstance(value, str)] + + +def _parse_optional_string(value: object) -> str | None: + if value is None: + return None + normalized = str(value).strip() + return normalized or None + + +def _parse_optional_int(value: object) -> int | None: + if isinstance(value, int): + return value + if isinstance(value, str) and value.strip(): + try: + return int(value) + except ValueError: + return None + return None + + +def _normalize_interests(value: object) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if isinstance(item, str)] + + +def _deserialize_candidate_payload(value: object) -> dict[str, object] | None: + if not isinstance(value, dict): + return None + scholar_id = str(value.get("scholar_id", "")).strip() + display_name = str(value.get("display_name", "")).strip() + profile_url = str(value.get("profile_url", "")).strip() + if not scholar_id or not display_name or not profile_url: + return None + return { + "scholar_id": scholar_id, + "display_name": display_name, + "affiliation": _parse_optional_string(value.get("affiliation")), + "email_domain": _parse_optional_string(value.get("email_domain")), + "cited_by_count": _parse_optional_int(value.get("cited_by_count")), + "interests": _normalize_interests(value.get("interests")), + "profile_url": profile_url, + "profile_image_url": _parse_optional_string(value.get("profile_image_url")), + } + + +def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]: + candidates_payload = payload.get("candidates") + if not isinstance(candidates_payload, list): + return [] + normalized: list[dict[str, object]] = [] + for value in candidates_payload: + candidate = _deserialize_candidate_payload(value) + if candidate is not None: + normalized.append(candidate) + return normalized + + +def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None: + if not isinstance(payload, dict): + return None + + state = _payload_state(payload) + if state is None: + return None + + marker_counts = _payload_marker_counts(payload) + warnings = _payload_warnings(payload) + from app.services.domains.scholar.parser import ScholarSearchCandidate + + normalized_candidates = _deserialize_candidates(payload) + + return ParsedAuthorSearchPage( + state=state, + state_reason=str(payload.get("state_reason", "")).strip() or "unknown", + candidates=[ + ScholarSearchCandidate( + scholar_id=item["scholar_id"], + display_name=item["display_name"], + affiliation=item["affiliation"], + email_domain=item["email_domain"], + cited_by_count=item["cited_by_count"], + interests=item["interests"], + profile_url=item["profile_url"], + profile_image_url=item["profile_image_url"], + ) + for item in normalized_candidates + ], + marker_counts=marker_counts, + warnings=warnings, + ) + + +async def _cache_get_author_search_result( + db_session: AsyncSession, + *, + query_key: str, + now_utc: datetime, +) -> ParsedAuthorSearchPage | None: + result = await db_session.execute( + select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) + ) + entry = result.scalar_one_or_none() + if entry is None: + return None + expires_at = entry.expires_at + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + if expires_at <= now_utc: + await db_session.delete(entry) + return None + parsed = _deserialize_parsed_author_search_page(entry.payload) + if parsed is None: + await db_session.delete(entry) + return None + return parsed + + +async def _cache_set_author_search_result( + db_session: AsyncSession, + *, + query_key: str, + parsed: ParsedAuthorSearchPage, + ttl_seconds: float, + max_entries: int, + now_utc: datetime, +) -> None: + ttl = max(float(ttl_seconds), 0.0) + existing_result = await db_session.execute( + select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) + ) + existing = existing_result.scalar_one_or_none() + + if ttl <= 0.0: + if existing is not None: + await db_session.delete(existing) + return + + expires_at = now_utc + timedelta(seconds=ttl) + payload = _serialize_parsed_author_search_page(parsed) + if existing is None: + db_session.add( + AuthorSearchCacheEntry( + query_key=query_key, + payload=payload, + expires_at=expires_at, + cached_at=now_utc, + updated_at=now_utc, + ) + ) + else: + existing.payload = payload + existing.expires_at = expires_at + existing.cached_at = now_utc + existing.updated_at = now_utc + + await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries) + + +async def _prune_author_search_cache( + db_session: AsyncSession, + *, + now_utc: datetime, + max_entries: int, +) -> None: + await db_session.execute( + delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc) + ) + bounded_max_entries = max(1, int(max_entries)) + count_result = await db_session.execute( + select(func.count()).select_from(AuthorSearchCacheEntry) + ) + entry_count = int(count_result.scalar_one() or 0) + overflow = max(0, entry_count - bounded_max_entries) + if overflow <= 0: + return + stale_keys_result = await db_session.execute( + select(AuthorSearchCacheEntry.query_key) + .order_by(AuthorSearchCacheEntry.cached_at.asc()) + .limit(overflow) + ) + stale_keys = [str(row[0]) for row in stale_keys_result.all()] + if stale_keys: + await db_session.execute( + delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys)) + ) + + +def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool: + return parsed.state == ParseState.BLOCKED_OR_CAPTCHA + + +def _author_search_cooldown_remaining_seconds( + runtime_state: AuthorSearchRuntimeState, + now_utc: datetime, +) -> int: + cooldown_until = runtime_state.cooldown_until + if cooldown_until is None: + return 0 + if cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + remaining_seconds = int((cooldown_until - now_utc).total_seconds()) + return max(0, remaining_seconds) + + +def _reset_author_search_runtime_state_for_tests() -> None: + # Runtime state now lives in the database; tests should reset via DB fixtures. + return None + + +async def list_scholars_for_user( + db_session: AsyncSession, + *, + user_id: int, +) -> list[ScholarProfile]: + result = await db_session.execute( + select(ScholarProfile) + .where(ScholarProfile.user_id == user_id) + .order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc()) + ) + return list(result.scalars().all()) + + +async def create_scholar_for_user( + db_session: AsyncSession, + *, + user_id: int, + scholar_id: str, + display_name: str, + profile_image_url: str | None = None, +) -> ScholarProfile: + profile = ScholarProfile( + user_id=user_id, + scholar_id=validate_scholar_id(scholar_id), + display_name=normalize_display_name(display_name), + profile_image_url=normalize_profile_image_url(profile_image_url), + ) + db_session.add(profile) + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise ScholarServiceError("That scholar is already tracked for this account.") from exc + await db_session.refresh(profile) + return profile + + +async def get_user_scholar_by_id( + db_session: AsyncSession, + *, + user_id: int, + scholar_profile_id: int, +) -> ScholarProfile | None: + result = await db_session.execute( + select(ScholarProfile).where( + ScholarProfile.id == scholar_profile_id, + ScholarProfile.user_id == user_id, + ) + ) + return result.scalar_one_or_none() + + +async def toggle_scholar_enabled( + db_session: AsyncSession, + *, + profile: ScholarProfile, +) -> ScholarProfile: + profile.is_enabled = not profile.is_enabled + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def delete_scholar( + db_session: AsyncSession, + *, + profile: ScholarProfile, + upload_dir: str | None = None, +) -> None: + if upload_dir: + upload_root = _ensure_upload_root(upload_dir, create=True) + _safe_remove_upload(upload_root, profile.profile_image_upload_path) + + await db_session.delete(profile) + await db_session.commit() + + +def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]: + normalized_query = query.strip() + if len(normalized_query) < 2: + raise ScholarServiceError("Search query must be at least 2 characters.") + bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) + return normalized_query, bounded_limit, normalized_query.casefold() + + +def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage: + logger.warning( + "scholar_search.disabled_by_configuration", + extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query}, + ) + return _policy_blocked_author_search_result( + reason=SEARCH_DISABLED_REASON, + warning_codes=["author_search_disabled_by_configuration"], + limit=bounded_limit, + ) + + +def _normalize_runtime_cooldown_state( + runtime_state: AuthorSearchRuntimeState, + *, + now_utc: datetime, +) -> bool: + if runtime_state.cooldown_until is None: + return False + cooldown_until = runtime_state.cooldown_until + updated = False + if cooldown_until.tzinfo is None: + cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) + runtime_state.cooldown_until = cooldown_until + updated = True + if now_utc < cooldown_until: + return updated + logger.info( + "scholar_search.cooldown_expired", + extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()}, + ) + runtime_state.cooldown_until = None + runtime_state.cooldown_rejection_count = 0 + runtime_state.cooldown_alert_emitted = False + return True + + +def _cooldown_warning_codes( + *, + runtime_state: AuthorSearchRuntimeState, + cooldown_remaining_seconds: int, +) -> list[str]: + warning_codes = [ + "author_search_cooldown_active", + f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s", + ] + if bool(runtime_state.cooldown_alert_emitted): + warning_codes.append("author_search_cooldown_alert_threshold_exceeded") + return warning_codes + + +def _emit_cooldown_threshold_alert( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + cooldown_rejection_alert_threshold: int, +) -> bool: + runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1 + threshold = max(1, int(cooldown_rejection_alert_threshold)) + if int(runtime_state.cooldown_rejection_count) < threshold: + return True + if bool(runtime_state.cooldown_alert_emitted): + return True + logger.error( + "scholar_search.cooldown_rejection_threshold_exceeded", + extra={ + "event": "scholar_search.cooldown_rejection_threshold_exceeded", + "query": normalized_query, + "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count), + "threshold": threshold, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + runtime_state.cooldown_alert_emitted = True + return True + + +def _cooldown_block_result( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + bounded_limit: int, + cooldown_rejection_alert_threshold: int, + cooldown_remaining_seconds: int, +) -> ParsedAuthorSearchPage: + _emit_cooldown_threshold_alert( + runtime_state=runtime_state, + normalized_query=normalized_query, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + ) + logger.warning( + "scholar_search.cooldown_active", + extra={ + "event": "scholar_search.cooldown_active", + "query": normalized_query, + "cooldown_remaining_seconds": cooldown_remaining_seconds, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + return _policy_blocked_author_search_result( + reason=SEARCH_COOLDOWN_REASON, + warning_codes=_cooldown_warning_codes( + runtime_state=runtime_state, + cooldown_remaining_seconds=cooldown_remaining_seconds, + ), + limit=bounded_limit, + ) + + +async def _cache_hit_result( + db_session: AsyncSession, + *, + query_key: str, + now_utc: datetime, + normalized_query: str, + bounded_limit: int, +) -> ParsedAuthorSearchPage | None: + cached = await _cache_get_author_search_result( + db_session, + query_key=query_key, + now_utc=now_utc, + ) + if cached is None: + return None + logger.info( + "scholar_search.cache_hit", + extra={ + "event": "scholar_search.cache_hit", + "query": normalized_query, + "state": cached.state.value, + "state_reason": cached.state_reason, + }, + ) + state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None + return _trim_author_search_result( + cached, + limit=bounded_limit, + extra_warnings=["author_search_served_from_cache"], + state_reason_override=state_reason_override, + ) + + +def _throttle_sleep_seconds( + *, + runtime_state: AuthorSearchRuntimeState, + now_utc: datetime, + min_interval_seconds: float, + interval_jitter_seconds: float, +) -> tuple[float, bool]: + updated = False + if runtime_state.last_live_request_at is None: + enforced_wait_seconds = 0.0 + else: + last_live_request_at = runtime_state.last_live_request_at + if last_live_request_at.tzinfo is None: + last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc) + runtime_state.last_live_request_at = last_live_request_at + updated = True + enforced_wait_seconds = ( + last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc + ).total_seconds() + jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0)) + return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated + + +async def _wait_for_author_search_throttle( + *, + runtime_state: AuthorSearchRuntimeState, + normalized_query: str, + now_utc: datetime, + min_interval_seconds: float, + interval_jitter_seconds: float, +) -> bool: + sleep_seconds, updated = _throttle_sleep_seconds( + runtime_state=runtime_state, + now_utc=now_utc, + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + ) + if sleep_seconds <= 0.0: + return updated + logger.info( + "scholar_search.throttle_wait", + extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)}, + ) + await asyncio.sleep(sleep_seconds) + return True + + +async def _fetch_author_search_with_retries( + *, + source: ScholarSource, + normalized_query: str, + network_error_retries: int, + retry_backoff_seconds: float, +) -> tuple[ParsedAuthorSearchPage, int, list[str]]: + max_attempts = max(1, int(network_error_retries) + 1) + parsed: ParsedAuthorSearchPage | None = None + retry_warnings: list[str] = [] + retry_scheduled_count = 0 + for attempt_index in range(max_attempts): + fetch_result = await source.fetch_author_search_html(normalized_query, start=0) + parsed = parse_author_search_page(fetch_result) + if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1: + break + retry_warnings.append("network_retry_scheduled_for_author_search") + retry_scheduled_count += 1 + retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index) + if retry_sleep_seconds > 0: + await asyncio.sleep(retry_sleep_seconds) + if parsed is None: + raise ScholarServiceError("Unable to complete scholar author search.") + return parsed, retry_scheduled_count, retry_warnings + + +def _with_retry_warnings( + parsed: ParsedAuthorSearchPage, + *, + retry_warnings: list[str], + retry_scheduled_count: int, + retry_alert_threshold: int, + normalized_query: str, +) -> ParsedAuthorSearchPage: + merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings)) + threshold = max(1, int(retry_alert_threshold)) + if retry_scheduled_count < threshold: + return merged + logger.warning( + "scholar_search.retry_threshold_exceeded", + extra={ + "event": "scholar_search.retry_threshold_exceeded", + "query": normalized_query, + "retry_scheduled_count": retry_scheduled_count, + "threshold": threshold, + "final_state": merged.state.value, + "final_state_reason": merged.state_reason, + }, + ) + return replace( + merged, + warnings=_merge_warnings( + merged.warnings, + [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"], + ), + ) + + +def _apply_block_circuit_breaker( + *, + runtime_state: AuthorSearchRuntimeState, + merged_parsed: ParsedAuthorSearchPage, + cooldown_block_threshold: int, + cooldown_seconds: int, + normalized_query: str, +) -> ParsedAuthorSearchPage: + if not _is_author_search_block_state(merged_parsed): + runtime_state.consecutive_blocked_count = 0 + return merged_parsed + runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1 + logger.warning( + "scholar_search.block_detected", + extra={ + "event": "scholar_search.block_detected", + "query": normalized_query, + "state_reason": merged_parsed.state_reason, + "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count), + }, + ) + if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)): + return merged_parsed + runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds))) + runtime_state.consecutive_blocked_count = 0 + runtime_state.cooldown_rejection_count = 0 + runtime_state.cooldown_alert_emitted = False + logger.error( + "scholar_search.cooldown_activated", + extra={ + "event": "scholar_search.cooldown_activated", + "query": normalized_query, + "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, + }, + ) + return replace( + merged_parsed, + warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]), + ) + + +def _resolve_author_search_cache_ttl_seconds( + *, + merged_parsed: ParsedAuthorSearchPage, + blocked_cache_ttl_seconds: int, + cache_ttl_seconds: int, +) -> int: + if _is_author_search_block_state(merged_parsed): + return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds))) + return max(1, int(cache_ttl_seconds)) + + +async def _load_locked_runtime_state( + db_session: AsyncSession, +) -> AuthorSearchRuntimeState: + await _acquire_author_search_lock(db_session) + return await _load_runtime_state_for_update(db_session) + + +async def _cooldown_or_cache_result( + db_session: AsyncSession, + *, + runtime_state: AuthorSearchRuntimeState, + query_key: str, + normalized_query: str, + bounded_limit: int, + cooldown_rejection_alert_threshold: int, +) -> tuple[ParsedAuthorSearchPage | None, bool]: + runtime_state_updated = _normalize_runtime_cooldown_state( + runtime_state, + now_utc=datetime.now(timezone.utc), + ) + cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds( + runtime_state, + datetime.now(timezone.utc), + ) + if cooldown_remaining_seconds > 0: + return ( + _cooldown_block_result( + runtime_state=runtime_state, + normalized_query=normalized_query, + bounded_limit=bounded_limit, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + cooldown_remaining_seconds=cooldown_remaining_seconds, + ), + True, + ) + cached_result = await _cache_hit_result( + db_session, + query_key=query_key, + now_utc=datetime.now(timezone.utc), + normalized_query=normalized_query, + bounded_limit=bounded_limit, + ) + return cached_result, runtime_state_updated + + +async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]: + runtime_state_updated = await _wait_for_author_search_throttle( + runtime_state=runtime_state, + normalized_query=normalized_query, + now_utc=datetime.now(timezone.utc), + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + ) + parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries( + source=source, + normalized_query=normalized_query, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + ) + runtime_state.last_live_request_at = datetime.now(timezone.utc) + merged = _with_retry_warnings( + parsed, + retry_warnings=retry_warnings, + retry_scheduled_count=retry_count, + retry_alert_threshold=retry_alert_threshold, + normalized_query=normalized_query, + ) + merged = _apply_block_circuit_breaker( + runtime_state=runtime_state, + merged_parsed=merged, + cooldown_block_threshold=cooldown_block_threshold, + cooldown_seconds=cooldown_seconds, + normalized_query=normalized_query, + ) + ttl_seconds = _resolve_author_search_cache_ttl_seconds( + merged_parsed=merged, + blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, + cache_ttl_seconds=cache_ttl_seconds, + ) + await _cache_set_author_search_result( + db_session, + query_key=query_key, + parsed=merged, + ttl_seconds=float(ttl_seconds), + max_entries=cache_max_entries, + now_utc=datetime.now(timezone.utc), + ) + return merged, True + + +async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage: + normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit) + if not search_enabled: + return _disabled_search_result( + normalized_query=normalized_query, + bounded_limit=bounded_limit, + ) + + runtime_state = await _load_locked_runtime_state(db_session) + early_result, runtime_state_updated = await _cooldown_or_cache_result( + db_session, + runtime_state=runtime_state, + query_key=query_key, + normalized_query=normalized_query, + bounded_limit=bounded_limit, + cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, + ) + if early_result is not None: + await db_session.commit() + return early_result + + merged_parsed, live_runtime_updated = await _perform_live_author_search( + db_session, + source=source, + runtime_state=runtime_state, + normalized_query=normalized_query, + query_key=query_key, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + min_interval_seconds=min_interval_seconds, + interval_jitter_seconds=interval_jitter_seconds, + retry_alert_threshold=retry_alert_threshold, + cooldown_block_threshold=cooldown_block_threshold, + cooldown_seconds=cooldown_seconds, + blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, + cache_ttl_seconds=cache_ttl_seconds, + cache_max_entries=cache_max_entries, + ) + runtime_state_updated = runtime_state_updated or live_runtime_updated + if runtime_state_updated: + await db_session.commit() + return _trim_author_search_result(merged_parsed, limit=bounded_limit) + + +async def hydrate_profile_metadata( + db_session: AsyncSession, + *, + profile: ScholarProfile, + source: ScholarSource, +) -> ScholarProfile: + fetch_result = await source.fetch_profile_html(profile.scholar_id) + parsed_page = parse_profile_page(fetch_result) + + if parsed_page.profile_name and not (profile.display_name or "").strip(): + profile.display_name = parsed_page.profile_name + if parsed_page.profile_image_url and not profile.profile_image_url: + profile.profile_image_url = parsed_page.profile_image_url + + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def set_profile_image_override_url( + db_session: AsyncSession, + *, + profile: ScholarProfile, + image_url: str | None, + upload_dir: str, +) -> ScholarProfile: + upload_root = _ensure_upload_root(upload_dir, create=True) + _safe_remove_upload(upload_root, profile.profile_image_upload_path) + + profile.profile_image_upload_path = None + profile.profile_image_override_url = normalize_profile_image_url(image_url) + + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def clear_profile_image_customization( + db_session: AsyncSession, + *, + profile: ScholarProfile, + upload_dir: str, +) -> ScholarProfile: + upload_root = _ensure_upload_root(upload_dir, create=True) + _safe_remove_upload(upload_root, profile.profile_image_upload_path) + + profile.profile_image_upload_path = None + profile.profile_image_override_url = None + + await db_session.commit() + await db_session.refresh(profile) + return profile + + +async def set_profile_image_upload( + db_session: AsyncSession, + *, + profile: ScholarProfile, + content_type: str | None, + image_bytes: bytes, + upload_dir: str, + max_upload_bytes: int, +) -> ScholarProfile: + normalized_content_type = (content_type or "").strip().lower() + extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type) + if extension is None: + raise ScholarServiceError( + "Unsupported image type. Use JPEG, PNG, WEBP, or GIF." + ) + + if not image_bytes: + raise ScholarServiceError("Uploaded image file is empty.") + + if len(image_bytes) > max_upload_bytes: + raise ScholarServiceError( + f"Uploaded image exceeds {max_upload_bytes} bytes." + ) + + upload_root = _ensure_upload_root(upload_dir, create=True) + user_dir = upload_root / str(profile.user_id) + user_dir.mkdir(parents=True, exist_ok=True) + + filename = f"{profile.id}_{uuid4().hex}{extension}" + relative_path = os.path.join(str(profile.user_id), filename) + absolute_path = _resolve_upload_path(upload_root, relative_path) + absolute_path.write_bytes(image_bytes) + + old_path = profile.profile_image_upload_path + profile.profile_image_upload_path = relative_path + profile.profile_image_override_url = None + + await db_session.commit() + await db_session.refresh(profile) + + if old_path and old_path != relative_path: + _safe_remove_upload(upload_root, old_path) + + return profile diff --git a/app/services/domains/scholars/constants.py b/app/services/domains/scholars/constants.py new file mode 100644 index 0000000..7ee18ad --- /dev/null +++ b/app/services/domains/scholars/constants.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import re + +SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$") +MAX_IMAGE_URL_LENGTH = 2048 +MAX_AUTHOR_SEARCH_LIMIT = 25 + +DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512 +DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300 +DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1 +DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800 +DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0 +DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0 +DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2 +DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3 + +AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global" +AUTHOR_SEARCH_LOCK_NAMESPACE = 3901 +AUTHOR_SEARCH_LOCK_KEY = 1 + +ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/webp": ".webp", + "image/gif": ".gif", +} + +SEARCH_DISABLED_REASON = "search_disabled_by_configuration" +SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks" +SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response" + +STATE_REASON_HINTS: dict[str, str] = { + SEARCH_DISABLED_REASON: ( + "Scholar name search is currently disabled by service policy. " + "Add scholars by profile URL or Scholar ID." + ), + SEARCH_COOLDOWN_REASON: ( + "Scholar name search is temporarily paused after repeated block responses. " + "Use Scholar URL/ID adds until cooldown expires." + ), + SEARCH_CACHED_BLOCK_REASON: ( + "A recent blocked response was cached to reduce traffic. " + "Retry later or add by Scholar URL/ID." + ), + "network_dns_resolution_failed": ( + "DNS resolution failed while reaching scholar.google.com. " + "Verify container DNS/network and retry." + ), + "network_timeout": ( + "Request timed out before Google Scholar responded. " + "Increase delay/backoff and retry." + ), + "network_tls_error": ( + "TLS handshake/certificate validation failed. " + "Verify outbound TLS/network configuration." + ), + "blocked_http_429_rate_limited": ( + "Google Scholar rate-limited the request. " + "Slow request cadence and retry later." + ), + "blocked_unusual_traffic_detected": ( + "Google Scholar flagged traffic as unusual. " + "Increase delay/jitter and reduce concurrent scraping." + ), + "blocked_accounts_redirect": ( + "Request was redirected to Google Account sign-in. " + "Treat as access block and retry later." + ), +} diff --git a/app/services/domains/scholars/exceptions.py b/app/services/domains/scholars/exceptions.py new file mode 100644 index 0000000..f1383e3 --- /dev/null +++ b/app/services/domains/scholars/exceptions.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +class ScholarServiceError(ValueError): + """Raised for expected scholar-management validation failures.""" diff --git a/app/services/domains/scholars/search_hints.py b/app/services/domains/scholars/search_hints.py new file mode 100644 index 0000000..f905c14 --- /dev/null +++ b/app/services/domains/scholars/search_hints.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from app.db.models import ScholarProfile +from app.services.domains.scholar.parser import ParseState, ParsedAuthorSearchPage +from app.services.domains.scholars.constants import ( + MAX_AUTHOR_SEARCH_LIMIT, + STATE_REASON_HINTS, +) + + +def resolve_profile_image( + profile: ScholarProfile, + *, + uploaded_image_url: str | None, +) -> tuple[str | None, str]: + if profile.profile_image_upload_path and uploaded_image_url: + return uploaded_image_url, "upload" + if profile.profile_image_override_url: + return profile.profile_image_override_url, "override" + if profile.profile_image_url: + return profile.profile_image_url, "scraped" + return None, "none" + + +def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None: + if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}: + return None + return STATE_REASON_HINTS.get(state_reason) + + +def _merge_warnings(base: list[str], extra: list[str]) -> list[str]: + if not extra: + return sorted(set(base)) + return sorted(set(base + extra)) + + +def _trim_author_search_result( + parsed: ParsedAuthorSearchPage, + *, + limit: int, + extra_warnings: list[str] | None = None, + state_reason_override: str | None = None, +) -> ParsedAuthorSearchPage: + bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) + return ParsedAuthorSearchPage( + state=parsed.state, + state_reason=state_reason_override or parsed.state_reason, + candidates=parsed.candidates[:bounded_limit], + marker_counts=parsed.marker_counts, + warnings=_merge_warnings(parsed.warnings, extra_warnings or []), + ) + + +def _policy_blocked_author_search_result( + *, + reason: str, + warning_codes: list[str], + limit: int, +) -> ParsedAuthorSearchPage: + _ = limit + return ParsedAuthorSearchPage( + state=ParseState.BLOCKED_OR_CAPTCHA, + state_reason=reason, + candidates=[], + marker_counts={}, + warnings=_merge_warnings([], warning_codes), + ) diff --git a/app/services/domains/scholars/uploads.py b/app/services/domains/scholars/uploads.py new file mode 100644 index 0000000..c33fd47 --- /dev/null +++ b/app/services/domains/scholars/uploads.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from pathlib import Path + +from app.services.domains.scholars.exceptions import ScholarServiceError + + +def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path: + root = Path(upload_dir).expanduser().resolve() + if create: + root.mkdir(parents=True, exist_ok=True) + return root + + +def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path: + candidate = (upload_root / relative_path).resolve() + if upload_root != candidate and upload_root not in candidate.parents: + raise ScholarServiceError("Invalid scholar image path.") + return candidate + + +def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None: + if not relative_path: + return + try: + file_path = _resolve_upload_path(upload_root, relative_path) + except ScholarServiceError: + return + + try: + if file_path.exists() and file_path.is_file(): + file_path.unlink() + except OSError: + return + + +def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path: + root = _ensure_upload_root(upload_dir, create=False) + return _resolve_upload_path(root, relative_path) diff --git a/app/services/domains/scholars/validators.py b/app/services/domains/scholars/validators.py new file mode 100644 index 0000000..1c79baa --- /dev/null +++ b/app/services/domains/scholars/validators.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from urllib.parse import urlparse + +from app.services.domains.scholars.constants import MAX_IMAGE_URL_LENGTH, SCHOLAR_ID_PATTERN +from app.services.domains.scholars.exceptions import ScholarServiceError + + +def validate_scholar_id(value: str) -> str: + scholar_id = value.strip() + if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id): + raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.") + return scholar_id + + +def normalize_display_name(value: str) -> str | None: + normalized = value.strip() + return normalized if normalized else None + + +def normalize_profile_image_url(value: str | None) -> str | None: + if value is None: + return None + + candidate = value.strip() + if not candidate: + return None + + if len(candidate) > MAX_IMAGE_URL_LENGTH: + raise ScholarServiceError( + f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer." + ) + + parsed = urlparse(candidate) + if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: + raise ScholarServiceError("Image URL must be an absolute http(s) URL.") + + return candidate diff --git a/app/services/domains/settings/__init__.py b/app/services/domains/settings/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/app/services/domains/settings/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/app/services/domains/settings/application.py b/app/services/domains/settings/application.py new file mode 100644 index 0000000..3548e44 --- /dev/null +++ b/app/services/domains/settings/application.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import UserSetting + + +class UserSettingsServiceError(ValueError): + """Raised for expected settings-validation failures.""" + + +NAV_PAGE_DASHBOARD = "dashboard" +NAV_PAGE_SCHOLARS = "scholars" +NAV_PAGE_PUBLICATIONS = "publications" +NAV_PAGE_SETTINGS = "settings" +NAV_PAGE_STYLE_GUIDE = "style-guide" +NAV_PAGE_RUNS = "runs" +NAV_PAGE_USERS = "users" + +ALLOWED_NAV_PAGES = ( + NAV_PAGE_DASHBOARD, + NAV_PAGE_SCHOLARS, + NAV_PAGE_PUBLICATIONS, + NAV_PAGE_SETTINGS, + NAV_PAGE_STYLE_GUIDE, + NAV_PAGE_RUNS, + NAV_PAGE_USERS, +) +REQUIRED_NAV_PAGES = ( + NAV_PAGE_DASHBOARD, + NAV_PAGE_SCHOLARS, + NAV_PAGE_SETTINGS, +) +DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES) +HARD_MIN_RUN_INTERVAL_MINUTES = 15 +HARD_MIN_REQUEST_DELAY_SECONDS = 2 + + +def resolve_run_interval_minimum(configured_minimum: int | None) -> int: + try: + parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES + except (TypeError, ValueError): + parsed = HARD_MIN_RUN_INTERVAL_MINUTES + return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed) + + +def resolve_request_delay_minimum(configured_minimum: int | None) -> int: + try: + parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS + except (TypeError, ValueError): + parsed = HARD_MIN_REQUEST_DELAY_SECONDS + return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed) + + +def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise UserSettingsServiceError("Check interval must be a whole number.") from exc + effective_minimum = resolve_run_interval_minimum(minimum) + if parsed < effective_minimum: + raise UserSettingsServiceError( + f"Check interval must be at least {effective_minimum} minutes." + ) + return parsed + + +def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise UserSettingsServiceError("Request delay must be a whole number.") from exc + effective_minimum = resolve_request_delay_minimum(minimum) + if parsed < effective_minimum: + raise UserSettingsServiceError( + f"Request delay must be at least {effective_minimum} seconds." + ) + return parsed + + +def parse_nav_visible_pages(value: object) -> list[str]: + if not isinstance(value, list): + raise UserSettingsServiceError("Navigation visibility must be a list of page ids.") + + deduped: list[str] = [] + seen: set[str] = set() + + for raw_page in value: + if not isinstance(raw_page, str): + raise UserSettingsServiceError("Navigation visibility entries must be strings.") + + page_id = raw_page.strip() + if page_id not in ALLOWED_NAV_PAGES: + raise UserSettingsServiceError(f"Unsupported navigation page id: {page_id}") + + if page_id in seen: + continue + + seen.add(page_id) + deduped.append(page_id) + + missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen] + if missing_required: + raise UserSettingsServiceError( + "Dashboard, Scholars, and Settings must remain visible." + ) + + return deduped + + +async def get_or_create_settings( + db_session: AsyncSession, + *, + user_id: int, +) -> UserSetting: + result = await db_session.execute( + select(UserSetting).where(UserSetting.user_id == user_id) + ) + settings = result.scalar_one_or_none() + if settings is not None: + return settings + + settings = UserSetting(user_id=user_id) + db_session.add(settings) + await db_session.commit() + await db_session.refresh(settings) + return settings + + +async def update_settings( + db_session: AsyncSession, + *, + settings: UserSetting, + auto_run_enabled: bool, + run_interval_minutes: int, + request_delay_seconds: int, + nav_visible_pages: list[str], +) -> UserSetting: + settings.auto_run_enabled = auto_run_enabled + settings.run_interval_minutes = run_interval_minutes + settings.request_delay_seconds = request_delay_seconds + settings.nav_visible_pages = nav_visible_pages + await db_session.commit() + await db_session.refresh(settings) + return settings diff --git a/app/services/domains/users/__init__.py b/app/services/domains/users/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/app/services/domains/users/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/app/services/domains/users/application.py b/app/services/domains/users/application.py new file mode 100644 index 0000000..7f019c4 --- /dev/null +++ b/app/services/domains/users/application.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import re + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import User + +EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + +class UserServiceError(ValueError): + """Raised for expected user-management validation failures.""" + + +def normalize_email(value: str) -> str: + return value.strip().lower() + + +def validate_email(value: str) -> str: + email = normalize_email(value) + if not EMAIL_PATTERN.fullmatch(email): + raise UserServiceError("Enter a valid email address.") + return email + + +def validate_password(value: str) -> str: + password = value.strip() + if len(password) < 8: + raise UserServiceError("Password must be at least 8 characters.") + return password + + +async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None: + result = await db_session.execute(select(User).where(User.id == user_id)) + return result.scalar_one_or_none() + + +async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None: + result = await db_session.execute( + select(User).where(User.email == normalize_email(email)) + ) + return result.scalar_one_or_none() + + +async def list_users(db_session: AsyncSession) -> list[User]: + result = await db_session.execute(select(User).order_by(User.email.asc())) + return list(result.scalars().all()) + + +async def create_user( + db_session: AsyncSession, + *, + email: str, + password_hash: str, + is_admin: bool, +) -> User: + user = User( + email=validate_email(email), + password_hash=password_hash, + is_admin=is_admin, + is_active=True, + ) + db_session.add(user) + try: + await db_session.commit() + except IntegrityError as exc: + await db_session.rollback() + raise UserServiceError("A user with that email already exists.") from exc + await db_session.refresh(user) + return user + + +async def set_user_active( + db_session: AsyncSession, + *, + user: User, + is_active: bool, +) -> User: + user.is_active = is_active + await db_session.commit() + await db_session.refresh(user) + return user + + +async def set_user_password_hash( + db_session: AsyncSession, + *, + user: User, + password_hash: str, +) -> User: + user.password_hash = password_hash + await db_session.commit() + await db_session.refresh(user) + return user + diff --git a/app/services/import_export.py b/app/services/import_export.py index 0c8a84b..ba710a6 100644 --- a/app/services/import_export.py +++ b/app/services/import_export.py @@ -1,667 +1,3 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import datetime, timezone -import hashlib -import re -from typing import Any - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import Publication, ScholarProfile, ScholarPublication -from app.services import scholars as scholar_service -from app.services.ingestion import build_publication_url, normalize_title - -EXPORT_SCHEMA_VERSION = 1 -MAX_IMPORT_SCHOLARS = 10_000 -MAX_IMPORT_PUBLICATIONS = 100_000 -WORD_RE = re.compile(r"[a-z0-9]+") -SHA256_RE = re.compile(r"^[0-9a-f]{64}$") - - -class ImportExportError(ValueError): - """Raised when import/export payload constraints are violated.""" - - -@dataclass(frozen=True) -class ImportedPublicationInput: - profile: ScholarProfile - title: str - year: int | None - citation_count: int - author_text: str | None - venue_text: str | None - cluster_id: str | None - pub_url: str | None - pdf_url: str | None - fingerprint: str - is_read: bool - - -def _normalize_optional_text(value: Any) -> str | None: - if value is None: - return None - normalized = str(value).strip() - return normalized or None - - -def _normalize_optional_year(value: Any) -> int | None: - if value is None: - return None - try: - year = int(value) - except (TypeError, ValueError): - return None - if year < 1500 or year > 3000: - return None - return year - - -def _normalize_citation_count(value: Any) -> int: - try: - parsed = int(value) - except (TypeError, ValueError): - return 0 - return max(0, parsed) - - -def _first_author_last_name(authors_text: str | None) -> str: - if not authors_text: - return "" - first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() - words = WORD_RE.findall(first_author) - if not words: - return "" - return words[-1] - - -def _first_venue_word(venue_text: str | None) -> str: - if not venue_text: - return "" - words = WORD_RE.findall(venue_text.lower()) - if not words: - return "" - return words[0] - - -def _build_fingerprint( - *, - title: str, - year: int | None, - author_text: str | None, - venue_text: str | None, -) -> str: - canonical = "|".join( - [ - normalize_title(title), - str(year) if year is not None else "", - _first_author_last_name(author_text), - _first_venue_word(venue_text), - ] - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def _resolve_fingerprint( - *, - title: str, - year: int | None, - author_text: str | None, - venue_text: str | None, - provided_fingerprint: Any, -) -> str: - normalized = _normalize_optional_text(provided_fingerprint) - if normalized and SHA256_RE.fullmatch(normalized.lower()): - return normalized.lower() - return _build_fingerprint( - title=title, - year=year, - author_text=author_text, - venue_text=venue_text, - ) - - -def _validate_import_sizes( - *, - scholars: list[dict[str, Any]], - publications: list[dict[str, Any]], -) -> None: - if len(scholars) > MAX_IMPORT_SCHOLARS: - raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).") - if len(publications) > MAX_IMPORT_PUBLICATIONS: - raise ImportExportError( - f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})." - ) - - -async def _load_user_scholar_map( - db_session: AsyncSession, - *, - user_id: int, -) -> dict[str, ScholarProfile]: - result = await db_session.execute( - select(ScholarProfile).where(ScholarProfile.user_id == user_id) - ) - profiles = list(result.scalars().all()) - return {profile.scholar_id: profile for profile in profiles} - - -def _apply_imported_scholar_values( - *, - profile: ScholarProfile, - display_name: str | None, - profile_image_override_url: str | None, - is_enabled: bool, -) -> bool: - updated = False - if display_name and profile.display_name != display_name: - profile.display_name = display_name - updated = True - if profile.profile_image_override_url != profile_image_override_url: - profile.profile_image_override_url = profile_image_override_url - updated = True - if bool(profile.is_enabled) != bool(is_enabled): - profile.is_enabled = bool(is_enabled) - updated = True - return updated - - -def _new_scholar_profile( - *, - user_id: int, - scholar_id: str, - display_name: str | None, - profile_image_override_url: str | None, - is_enabled: bool, -) -> ScholarProfile: - return ScholarProfile( - user_id=user_id, - scholar_id=scholar_id, - display_name=display_name, - profile_image_override_url=profile_image_override_url, - is_enabled=bool(is_enabled), - ) - - -async def _upsert_imported_scholars( - db_session: AsyncSession, - *, - user_id: int, - scholars: list[dict[str, Any]], -) -> tuple[dict[str, ScholarProfile], dict[str, int]]: - scholar_map = await _load_user_scholar_map(db_session, user_id=user_id) - counters = {"scholars_created": 0, "scholars_updated": 0, "skipped_records": 0} - for item in scholars: - try: - scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"])) - display_name = scholar_service.normalize_display_name(str(item.get("display_name") or "")) - override_url = scholar_service.normalize_profile_image_url( - _normalize_optional_text(item.get("profile_image_override_url")) - ) - except (KeyError, scholar_service.ScholarServiceError): - counters["skipped_records"] += 1 - continue - is_enabled = bool(item.get("is_enabled", True)) - existing = scholar_map.get(scholar_id) - if existing is None: - profile = _new_scholar_profile( - user_id=user_id, - scholar_id=scholar_id, - display_name=display_name, - profile_image_override_url=override_url, - is_enabled=is_enabled, - ) - db_session.add(profile) - scholar_map[scholar_id] = profile - counters["scholars_created"] += 1 - continue - if _apply_imported_scholar_values( - profile=existing, - display_name=display_name, - profile_image_override_url=override_url, - is_enabled=is_enabled, - ): - counters["scholars_updated"] += 1 - await db_session.flush() - return scholar_map, counters - - -async def _find_publication_by_cluster( - db_session: AsyncSession, - *, - cluster_id: str, -) -> Publication | None: - result = await db_session.execute( - select(Publication).where(Publication.cluster_id == cluster_id) - ) - return result.scalar_one_or_none() - - -async def _find_publication_by_fingerprint( - db_session: AsyncSession, - *, - fingerprint_sha256: str, -) -> Publication | None: - result = await db_session.execute( - select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256) - ) - return result.scalar_one_or_none() - - -def _apply_imported_publication_values( - *, - publication: Publication, - title: str, - year: int | None, - citation_count: int, - author_text: str | None, - venue_text: str | None, - pub_url: str | None, - pdf_url: str | None, - cluster_id: str | None, -) -> bool: - updated = False - if cluster_id and publication.cluster_id != cluster_id: - publication.cluster_id = cluster_id - updated = True - if publication.title_raw != title: - publication.title_raw = title - publication.title_normalized = normalize_title(title) - updated = True - if publication.year != year: - publication.year = year - updated = True - if int(publication.citation_count or 0) != citation_count: - publication.citation_count = citation_count - updated = True - if publication.author_text != author_text: - publication.author_text = author_text - updated = True - if publication.venue_text != venue_text: - publication.venue_text = venue_text - updated = True - if pub_url and publication.pub_url != pub_url: - publication.pub_url = pub_url - updated = True - if pdf_url and publication.pdf_url != pdf_url: - publication.pdf_url = pdf_url - updated = True - return updated - - -def _new_publication( - *, - cluster_id: str | None, - fingerprint_sha256: str, - title: str, - year: int | None, - citation_count: int, - author_text: str | None, - venue_text: str | None, - pub_url: str | None, - pdf_url: str | None, -) -> Publication: - return Publication( - cluster_id=cluster_id, - fingerprint_sha256=fingerprint_sha256, - title_raw=title, - title_normalized=normalize_title(title), - year=year, - citation_count=citation_count, - author_text=author_text, - venue_text=venue_text, - pub_url=pub_url, - pdf_url=pdf_url, - ) - - -async def _resolve_publication_for_import( - db_session: AsyncSession, - *, - cluster_id: str | None, - fingerprint_sha256: str, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], -) -> Publication | None: - if cluster_id: - if cluster_id not in cluster_cache: - cluster_cache[cluster_id] = await _find_publication_by_cluster( - db_session, - cluster_id=cluster_id, - ) - if cluster_cache[cluster_id] is not None: - return cluster_cache[cluster_id] - if fingerprint_sha256 not in fingerprint_cache: - fingerprint_cache[fingerprint_sha256] = await _find_publication_by_fingerprint( - db_session, - fingerprint_sha256=fingerprint_sha256, - ) - return fingerprint_cache[fingerprint_sha256] - - -async def _upsert_scholar_publication_link( - db_session: AsyncSession, - *, - scholar_profile_id: int, - publication_id: int, - is_read: bool, -) -> tuple[bool, bool]: - result = await db_session.execute( - select(ScholarPublication).where( - ScholarPublication.scholar_profile_id == scholar_profile_id, - ScholarPublication.publication_id == publication_id, - ) - ) - link = result.scalar_one_or_none() - if link is None: - db_session.add( - ScholarPublication( - scholar_profile_id=scholar_profile_id, - publication_id=publication_id, - is_read=bool(is_read), - ) - ) - return True, False - if bool(link.is_read) == bool(is_read): - return False, False - link.is_read = bool(is_read) - return False, True - - -def _initialize_import_counters(counters: dict[str, int]) -> None: - counters.update( - { - "publications_created": 0, - "publications_updated": 0, - "links_created": 0, - "links_updated": 0, - } - ) - - -def _build_imported_publication_input( - *, - item: dict[str, Any], - scholar_map: dict[str, ScholarProfile], -) -> ImportedPublicationInput | None: - scholar_id = _normalize_optional_text(item.get("scholar_id")) - title = _normalize_optional_text(item.get("title")) - if not scholar_id or not title: - return None - profile = scholar_map.get(scholar_id) - if profile is None: - return None - year = _normalize_optional_year(item.get("year")) - author_text = _normalize_optional_text(item.get("author_text")) - venue_text = _normalize_optional_text(item.get("venue_text")) - return ImportedPublicationInput( - profile=profile, - title=title, - year=year, - citation_count=_normalize_citation_count(item.get("citation_count")), - author_text=author_text, - venue_text=venue_text, - cluster_id=_normalize_optional_text(item.get("cluster_id")), - pub_url=build_publication_url(_normalize_optional_text(item.get("pub_url"))), - pdf_url=build_publication_url(_normalize_optional_text(item.get("pdf_url"))), - fingerprint=_resolve_fingerprint( - title=title, - year=year, - author_text=author_text, - venue_text=venue_text, - provided_fingerprint=item.get("fingerprint_sha256"), - ), - is_read=bool(item.get("is_read", False)), - ) - - -def _update_link_counters( - *, - counters: dict[str, int], - link_created: bool, - link_updated: bool, -) -> None: - if link_created: - counters["links_created"] += 1 - if link_updated: - counters["links_updated"] += 1 - - -def _cache_resolved_publication( - *, - publication: Publication, - cluster_id: str | None, - fingerprint_sha256: str, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], -) -> None: - if cluster_id: - cluster_cache[cluster_id] = publication - fingerprint_cache[fingerprint_sha256] = publication - - -async def _create_import_publication( - db_session: AsyncSession, - *, - payload: ImportedPublicationInput, -) -> Publication: - publication = _new_publication( - cluster_id=payload.cluster_id, - fingerprint_sha256=payload.fingerprint, - title=payload.title, - year=payload.year, - citation_count=payload.citation_count, - author_text=payload.author_text, - venue_text=payload.venue_text, - pub_url=payload.pub_url, - pdf_url=payload.pdf_url, - ) - db_session.add(publication) - await db_session.flush() - return publication - - -def _update_import_publication( - *, - publication: Publication, - payload: ImportedPublicationInput, -) -> bool: - return _apply_imported_publication_values( - publication=publication, - title=payload.title, - year=payload.year, - citation_count=payload.citation_count, - author_text=payload.author_text, - venue_text=payload.venue_text, - pub_url=payload.pub_url, - pdf_url=payload.pdf_url, - cluster_id=payload.cluster_id, - ) - - -async def _upsert_publication_entity( - db_session: AsyncSession, - *, - payload: ImportedPublicationInput, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], -) -> tuple[Publication, bool, bool]: - publication = await _resolve_publication_for_import( - db_session, - cluster_id=payload.cluster_id, - fingerprint_sha256=payload.fingerprint, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - ) - created = False - updated = False - if publication is None: - publication = await _create_import_publication( - db_session, - payload=payload, - ) - created = True - else: - updated = _update_import_publication( - publication=publication, - payload=payload, - ) - _cache_resolved_publication( - publication=publication, - cluster_id=payload.cluster_id, - fingerprint_sha256=payload.fingerprint, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - ) - return publication, created, updated - - -async def _upsert_imported_publication( - db_session: AsyncSession, - *, - payload: ImportedPublicationInput, - cluster_cache: dict[str, Publication | None], - fingerprint_cache: dict[str, Publication | None], - counters: dict[str, int], -) -> None: - publication, created, updated = await _upsert_publication_entity( - db_session, - payload=payload, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - ) - if created: - counters["publications_created"] += 1 - if updated: - counters["publications_updated"] += 1 - link_created, link_updated = await _upsert_scholar_publication_link( - db_session, - scholar_profile_id=int(payload.profile.id), - publication_id=int(publication.id), - is_read=payload.is_read, - ) - _update_link_counters( - counters=counters, - link_created=link_created, - link_updated=link_updated, - ) - - -def _exported_at_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat() - - -def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]: - return { - "scholar_id": profile.scholar_id, - "display_name": profile.display_name, - "is_enabled": bool(profile.is_enabled), - "profile_image_override_url": profile.profile_image_override_url, - } - - -def _serialize_export_publication(row: tuple[Any, ...]) -> dict[str, Any]: - ( - scholar_id, - cluster_id, - fingerprint_sha256, - title_raw, - year, - citation_count, - author_text, - venue_text, - pub_url, - pdf_url, - is_read, - ) = row - return { - "scholar_id": scholar_id, - "cluster_id": cluster_id, - "fingerprint_sha256": fingerprint_sha256, - "title": title_raw, - "year": year, - "citation_count": int(citation_count or 0), - "author_text": author_text, - "venue_text": venue_text, - "pub_url": pub_url, - "pdf_url": pdf_url, - "is_read": bool(is_read), - } - - -async def export_user_data( - db_session: AsyncSession, - *, - user_id: int, -) -> dict[str, Any]: - scholars_result = await db_session.execute( - select(ScholarProfile) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarProfile.id.asc()) - ) - publication_result = await db_session.execute( - select( - ScholarProfile.scholar_id, - Publication.cluster_id, - Publication.fingerprint_sha256, - Publication.title_raw, - Publication.year, - Publication.citation_count, - Publication.author_text, - Publication.venue_text, - Publication.pub_url, - Publication.pdf_url, - ScholarPublication.is_read, - ) - .join(ScholarPublication, ScholarPublication.scholar_profile_id == ScholarProfile.id) - .join(Publication, Publication.id == ScholarPublication.publication_id) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) - ) - scholars = [_serialize_export_scholar(profile) for profile in scholars_result.scalars().all()] - publications = [ - _serialize_export_publication(row) - for row in publication_result.all() - ] - return { - "schema_version": EXPORT_SCHEMA_VERSION, - "exported_at": _exported_at_iso(), - "scholars": scholars, - "publications": publications, - } - - -async def import_user_data( - db_session: AsyncSession, - *, - user_id: int, - scholars: list[dict[str, Any]], - publications: list[dict[str, Any]], -) -> dict[str, int]: - _validate_import_sizes(scholars=scholars, publications=publications) - scholar_map, counters = await _upsert_imported_scholars( - db_session, - user_id=user_id, - scholars=scholars, - ) - cluster_cache: dict[str, Publication | None] = {} - fingerprint_cache: dict[str, Publication | None] = {} - _initialize_import_counters(counters) - for item in publications: - parsed_item = _build_imported_publication_input( - item=item, - scholar_map=scholar_map, - ) - if parsed_item is None: - counters["skipped_records"] += 1 - continue - await _upsert_imported_publication( - db_session, - payload=parsed_item, - cluster_cache=cluster_cache, - fingerprint_cache=fingerprint_cache, - counters=counters, - ) - await db_session.commit() - return counters +from app.services.domains.portability.application import * diff --git a/app/services/ingestion.py b/app/services/ingestion.py index 91fac24..2b2bcc9 100644 --- a/app/services/ingestion.py +++ b/app/services/ingestion.py @@ -1,2326 +1,3 @@ from __future__ import annotations -import asyncio -from dataclasses import dataclass, field -from datetime import datetime, timezone -import hashlib -import json -import logging -import re -from typing import Any -from urllib.parse import urljoin - -from sqlalchemy import select, text -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ( - CrawlRun, - Publication, - RunStatus, - RunTriggerType, - ScholarProfile, - ScholarPublication, -) -from app.services import continuation_queue as queue_service -from app.services import run_safety as run_safety_service -from app.services import user_settings as user_settings_service -from app.services.scholar_parser import ( - ParseState, - ParsedProfilePage, - PublicationCandidate, - parse_profile_page, -) -from app.services.scholar_source import FetchResult, ScholarSource -from app.settings import settings - -TITLE_ALNUM_RE = re.compile(r"[^a-z0-9]+") -WORD_RE = re.compile(r"[a-z0-9]+") -HTML_TAG_RE = re.compile(r"<[^>]+>", re.S) -SPACE_RE = re.compile(r"\s+") -FAILED_STATES = { - ParseState.BLOCKED_OR_CAPTCHA.value, - ParseState.LAYOUT_CHANGED.value, - ParseState.NETWORK_ERROR.value, - "ingestion_error", -} -FAILURE_BUCKET_BLOCKED = "blocked_or_captcha" -FAILURE_BUCKET_NETWORK = "network_error" -FAILURE_BUCKET_LAYOUT = "layout_changed" -FAILURE_BUCKET_INGESTION = "ingestion_error" -FAILURE_BUCKET_OTHER = "other_failure" -RUN_LOCK_NAMESPACE = 8217 -RESUMABLE_PARTIAL_REASONS = { - "max_pages_reached", - "pagination_cursor_stalled", -} -RESUMABLE_PARTIAL_REASON_PREFIXES = ("page_state_network_error",) -INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS = 30 -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class RunExecutionSummary: - crawl_run_id: int - status: RunStatus - scholar_count: int - succeeded_count: int - failed_count: int - partial_count: int - new_publication_count: int - - -@dataclass(frozen=True) -class PagedParseResult: - fetch_result: FetchResult - parsed_page: ParsedProfilePage - first_page_fetch_result: FetchResult - first_page_parsed_page: ParsedProfilePage - first_page_fingerprint_sha256: str | None - publications: list[PublicationCandidate] - attempt_log: list[dict[str, Any]] - page_logs: list[dict[str, Any]] - pages_fetched: int - pages_attempted: int - has_more_remaining: bool - pagination_truncated_reason: str | None - continuation_cstart: int | None - skipped_no_change: bool - - -@dataclass -class RunProgress: - succeeded_count: int = 0 - failed_count: int = 0 - partial_count: int = 0 - scholar_results: list[dict[str, Any]] = field(default_factory=list) - - -@dataclass(frozen=True) -class ScholarProcessingOutcome: - result_entry: dict[str, Any] - succeeded_count_delta: int - failed_count_delta: int - partial_count_delta: int - discovered_publication_count: int - - -@dataclass(frozen=True) -class RunFailureSummary: - failed_state_counts: dict[str, int] - failed_reason_counts: dict[str, int] - scrape_failure_counts: dict[str, int] - retries_scheduled_count: int - scholars_with_retries_count: int - retry_exhausted_count: int - - -@dataclass(frozen=True) -class RunAlertSummary: - blocked_failure_count: int - network_failure_count: int - blocked_failure_threshold: int - network_failure_threshold: int - retry_scheduled_threshold: int - alert_flags: dict[str, bool] - - -@dataclass -class PagedLoopState: - fetch_result: FetchResult - parsed_page: ParsedProfilePage - attempt_log: list[dict[str, Any]] - page_logs: list[dict[str, Any]] - publications: list[PublicationCandidate] - pages_fetched: int - pages_attempted: int - current_cstart: int - next_cstart: int - has_more_remaining: bool = False - pagination_truncated_reason: str | None = None - continuation_cstart: int | None = None - - -class RunAlreadyInProgressError(RuntimeError): - """Raised when a run lock for a user is already held by another process.""" - - -class RunBlockedBySafetyPolicyError(RuntimeError): - def __init__( - self, - *, - code: str, - message: str, - safety_state: dict[str, Any], - ) -> None: - super().__init__(message) - self.code = code - self.message = message - self.safety_state = safety_state - - -def _int_or_default(value: Any, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _classify_failure_bucket(*, state: str, state_reason: str) -> str: - reason = state_reason.strip().lower() - normalized_state = state.strip().lower() - - if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"): - return FAILURE_BUCKET_BLOCKED - if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"): - return FAILURE_BUCKET_NETWORK - if normalized_state == ParseState.LAYOUT_CHANGED.value: - return FAILURE_BUCKET_LAYOUT - if normalized_state == "ingestion_error": - return FAILURE_BUCKET_INGESTION - return FAILURE_BUCKET_OTHER - - -class ScholarIngestionService: - def __init__(self, *, source: ScholarSource) -> None: - self._source = source - - async def _load_user_settings_for_run( - self, - db_session: AsyncSession, - *, - user_id: int, - trigger_type: RunTriggerType, - ): - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=user_id, - ) - await self._enforce_safety_gate( - db_session, - user_settings=user_settings, - user_id=user_id, - trigger_type=trigger_type, - ) - return user_settings - - async def _enforce_safety_gate( - self, - db_session: AsyncSession, - *, - user_settings, - user_id: int, - trigger_type: RunTriggerType, - ) -> None: - now_utc = datetime.now(timezone.utc) - previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc) - if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc): - await db_session.commit() - await db_session.refresh(user_settings) - logger.info( - "ingestion.safety_cooldown_cleared", - extra={ - "event": "ingestion.safety_cooldown_cleared", - "user_id": user_id, - "reason": previous.get("cooldown_reason"), - "cooldown_until": previous.get("cooldown_until"), - "metric_name": "ingestion_safety_cooldown_cleared_total", - "metric_value": 1, - }, - ) - now_utc = datetime.now(timezone.utc) - if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc): - await self._raise_safety_blocked_start( - db_session, - user_settings=user_settings, - user_id=user_id, - trigger_type=trigger_type, - now_utc=now_utc, - ) - - async def _raise_safety_blocked_start( - self, - db_session: AsyncSession, - *, - user_settings, - user_id: int, - trigger_type: RunTriggerType, - now_utc: datetime, - ) -> None: - safety_state = run_safety_service.register_cooldown_blocked_start( - user_settings, - now_utc=now_utc, - ) - await db_session.commit() - logger.warning( - "ingestion.safety_policy_blocked_run_start", - extra={ - "event": "ingestion.safety_policy_blocked_run_start", - "user_id": user_id, - "trigger_type": trigger_type.value, - "reason": safety_state.get("cooldown_reason"), - "cooldown_until": safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), - "blocked_start_count": ((safety_state.get("counters") or {}).get("blocked_start_count")), - "metric_name": "ingestion_safety_run_start_blocked_total", - "metric_value": 1, - }, - ) - raise RunBlockedBySafetyPolicyError( - code="scrape_cooldown_active", - message="Scrape safety cooldown is active; run start is temporarily blocked.", - safety_state=safety_state, - ) - - @staticmethod - def _normalize_run_targets( - *, - scholar_profile_ids: set[int] | None, - start_cstart_by_scholar_id: dict[int, int] | None, - ) -> tuple[set[int] | None, dict[int, int]]: - filtered_scholar_ids = ( - {int(value) for value in scholar_profile_ids} - if scholar_profile_ids is not None - else None - ) - start_cstart_map = { - int(key): max(0, int(value)) - for key, value in (start_cstart_by_scholar_id or {}).items() - } - return filtered_scholar_ids, start_cstart_map - - async def _load_target_scholars( - self, - db_session: AsyncSession, - *, - user_id: int, - filtered_scholar_ids: set[int] | None, - ) -> list[ScholarProfile]: - scholars_stmt = ( - select(ScholarProfile) - .where(ScholarProfile.user_id == user_id, ScholarProfile.is_enabled.is_(True)) - .order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc()) - ) - if filtered_scholar_ids is not None: - scholars_stmt = scholars_stmt.where(ScholarProfile.id.in_(filtered_scholar_ids)) - scholars_result = await db_session.execute(scholars_stmt) - scholars = list(scholars_result.scalars().all()) - await self._clear_missing_filtered_jobs( - db_session, - user_id=user_id, - filtered_scholar_ids=filtered_scholar_ids, - scholars=scholars, - ) - return scholars - - async def _clear_missing_filtered_jobs( - self, - db_session: AsyncSession, - *, - user_id: int, - filtered_scholar_ids: set[int] | None, - scholars: list[ScholarProfile], - ) -> None: - if filtered_scholar_ids is None: - return - found_ids = {int(scholar.id) for scholar in scholars} - missing_ids = filtered_scholar_ids - found_ids - for scholar_profile_id in missing_ids: - await queue_service.clear_job_for_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - ) - - @staticmethod - def _create_running_run( - *, - user_id: int, - trigger_type: RunTriggerType, - scholar_count: int, - idempotency_key: str | None, - ) -> CrawlRun: - return CrawlRun( - user_id=user_id, - trigger_type=trigger_type, - status=RunStatus.RUNNING, - scholar_count=scholar_count, - new_pub_count=0, - idempotency_key=idempotency_key, - error_log={}, - ) - - @staticmethod - def _log_run_started( - *, - user_id: int, - trigger_type: RunTriggerType, - scholar_count: int, - filtered: bool, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> None: - logger.info( - "ingestion.run_started", - extra={ - "event": "ingestion.run_started", - "user_id": user_id, - "trigger_type": trigger_type.value, - "scholar_count": scholar_count, - "is_filtered_run": filtered, - "request_delay_seconds": request_delay_seconds, - "network_error_retries": network_error_retries, - "retry_backoff_seconds": retry_backoff_seconds, - "max_pages_per_scholar": max_pages_per_scholar, - "page_size": page_size, - "idempotency_key": idempotency_key, - "alert_blocked_failure_threshold": alert_blocked_failure_threshold, - "alert_network_failure_threshold": alert_network_failure_threshold, - "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, - }, - ) - - @staticmethod - async def _wait_between_scholars(*, index: int, request_delay_seconds: int) -> None: - if index <= 0 or request_delay_seconds <= 0: - return - await asyncio.sleep(float(request_delay_seconds)) - - @staticmethod - def _assert_valid_paged_parse_result( - *, - scholar_id: str, - paged_parse_result: PagedParseResult, - ) -> None: - parsed_page = paged_parse_result.parsed_page - if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS}: - if any(code.startswith("layout_") for code in parsed_page.warnings): - raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") - for publication in paged_parse_result.publications: - if not publication.title.strip(): - raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") - if publication.citation_count is not None and int(publication.citation_count) < 0: - raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") - - @staticmethod - def _apply_first_page_profile_metadata( - *, - scholar: ScholarProfile, - paged_parse_result: PagedParseResult, - run_dt: datetime, - ) -> None: - first_page = paged_parse_result.first_page_parsed_page - if first_page.profile_name and not (scholar.display_name or "").strip(): - scholar.display_name = first_page.profile_name - if first_page.profile_image_url: - scholar.profile_image_url = first_page.profile_image_url - if paged_parse_result.first_page_fingerprint_sha256: - scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 - scholar.last_initial_page_checked_at = run_dt - - @staticmethod - def _log_scholar_parsed( - *, - user_id: int, - run_id: int, - scholar: ScholarProfile, - paged_parse_result: PagedParseResult, - ) -> None: - parsed_page = paged_parse_result.parsed_page - logger.info( - "ingestion.scholar_parsed", - extra={ - "event": "ingestion.scholar_parsed", - "user_id": user_id, - "crawl_run_id": run_id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "publication_count": len(paged_parse_result.publications), - "has_show_more_button": parsed_page.has_show_more_button, - "pages_fetched": paged_parse_result.pages_fetched, - "pages_attempted": paged_parse_result.pages_attempted, - "has_more_remaining": paged_parse_result.has_more_remaining, - "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "warning_count": len(parsed_page.warnings), - "skipped_no_change": paged_parse_result.skipped_no_change, - }, - ) - - @staticmethod - def _build_result_entry( - *, - scholar: ScholarProfile, - start_cstart: int, - paged_parse_result: PagedParseResult, - ) -> dict[str, Any]: - parsed_page = paged_parse_result.parsed_page - return { - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "outcome": "failed", - "attempt_count": len(paged_parse_result.attempt_log), - "publication_count": len(paged_parse_result.publications), - "start_cstart": start_cstart, - "articles_range": parsed_page.articles_range, - "warnings": parsed_page.warnings, - "has_show_more_button": parsed_page.has_show_more_button, - "pages_fetched": paged_parse_result.pages_fetched, - "pages_attempted": paged_parse_result.pages_attempted, - "has_more_remaining": paged_parse_result.has_more_remaining, - "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "continuation_cstart": paged_parse_result.continuation_cstart, - "skipped_no_change": paged_parse_result.skipped_no_change, - "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - } - - def _skipped_no_change_outcome( - self, - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - first_page = paged_parse_result.first_page_parsed_page - scholar.last_run_status = RunStatus.SUCCESS - scholar.last_run_dt = run_dt - result_entry["state"] = first_page.state.value - result_entry["state_reason"] = "no_change_initial_page_signature" - result_entry["outcome"] = "success" - result_entry["publication_count"] = 0 - result_entry["warnings"] = first_page.warnings - result_entry["debug"] = { - "state_reason": "no_change_initial_page_signature", - "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - "attempt_log": paged_parse_result.attempt_log, - "page_logs": paged_parse_result.page_logs, - } - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=0, - discovered_publication_count=0, - ) - - async def _upsert_publications_outcome( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - parsed_page = paged_parse_result.parsed_page - publications = paged_parse_result.publications - had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} - has_partial_set = len(publications) > 0 and had_page_failure - if (not had_page_failure) or has_partial_set: - return await self._upsert_success_or_exception( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_set, - ) - return self._parse_failure_outcome( - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - - async def _upsert_success_or_exception( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, - ) -> ScholarProcessingOutcome: - try: - return await self._upsert_success( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_publication_set, - ) - except Exception as exc: - return self._upsert_exception_outcome( - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - exc=exc, - ) - - async def _upsert_success( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, - ) -> ScholarProcessingOutcome: - discovered_count = await self._upsert_profile_publications( - db_session, - run=run, - scholar=scholar, - publications=paged_parse_result.publications, - ) - is_partial = ( - paged_parse_result.has_more_remaining - or paged_parse_result.pagination_truncated_reason is not None - or has_partial_publication_set - ) - scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS - scholar.last_run_dt = run_dt - result_entry["outcome"] = "partial" if is_partial else "success" - if is_partial: - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=1 if is_partial else 0, - discovered_publication_count=discovered_count, - ) - - def _upsert_exception_outcome( - self, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - exc: Exception, - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["state"] = "ingestion_error" - result_entry["state_reason"] = "publication_upsert_exception" - result_entry["outcome"] = "failed" - result_entry["error"] = str(exc) - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - exception=exc, - ) - logger.exception( - "ingestion.scholar_failed", - extra={ - "event": "ingestion.scholar_failed", - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - def _parse_failure_outcome( - self, - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - logger.warning( - "ingestion.scholar_parse_failed", - extra={ - "event": "ingestion.scholar_parse_failed", - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": paged_parse_result.parsed_page.state.value, - "state_reason": paged_parse_result.parsed_page.state_reason, - "status_code": paged_parse_result.fetch_result.status_code, - }, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - async def _sync_continuation_queue( - self, - db_session: AsyncSession, - *, - user_id: int, - scholar: ScholarProfile, - run: CrawlRun, - start_cstart: int, - result_entry: dict[str, Any], - paged_parse_result: PagedParseResult, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> None: - queue_reason, queue_cstart = self._resolve_continuation_queue_target( - outcome=str(result_entry.get("outcome", "")), - state=str(result_entry.get("state", "")), - pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, - continuation_cstart=paged_parse_result.continuation_cstart, - fallback_cstart=start_cstart, - ) - if auto_queue_continuations and queue_reason is not None: - await queue_service.upsert_job( - db_session, - user_id=user_id, - scholar_profile_id=scholar.id, - resume_cstart=queue_cstart, - reason=queue_reason, - run_id=run.id, - delay_seconds=queue_delay_seconds, - ) - result_entry["continuation_enqueued"] = True - result_entry["continuation_reason"] = queue_reason - result_entry["continuation_cstart"] = queue_cstart - return - if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id): - result_entry["continuation_cleared"] = True - - async def _process_scholar( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - start_cstart: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> ScholarProcessingOutcome: - try: - return await self._process_scholar_inner( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - start_cstart=start_cstart, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - except Exception as exc: - return self._unexpected_scholar_exception_outcome( - run=run, - scholar=scholar, - start_cstart=start_cstart, - exc=exc, - ) - - async def _process_scholar_inner( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - start_cstart: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> ScholarProcessingOutcome: - run_dt, paged_parse_result, result_entry = await self._fetch_and_prepare_scholar_result( - run=run, - scholar=scholar, - user_id=user_id, - start_cstart=start_cstart, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - ) - outcome = await self._resolve_scholar_outcome( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - await self._sync_continuation_queue( - db_session, - user_id=user_id, - scholar=scholar, - run=run, - start_cstart=start_cstart, - result_entry=outcome.result_entry, - paged_parse_result=paged_parse_result, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - return outcome - - async def _fetch_and_prepare_scholar_result( - self, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - start_cstart: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - ) -> tuple[datetime, PagedParseResult, dict[str, Any]]: - run_dt = datetime.now(timezone.utc) - paged_parse_result = await self._fetch_and_parse_all_pages_with_retry( - scholar_id=scholar.scholar_id, - start_cstart=start_cstart, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages=max_pages_per_scholar, - page_size=page_size, - previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, - ) - self._assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result) - self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt) - self._log_scholar_parsed(user_id=user_id, run_id=run.id, scholar=scholar, paged_parse_result=paged_parse_result) - result_entry = self._build_result_entry( - scholar=scholar, - start_cstart=start_cstart, - paged_parse_result=paged_parse_result, - ) - return run_dt, paged_parse_result, result_entry - - async def _resolve_scholar_outcome( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - if paged_parse_result.skipped_no_change: - return self._skipped_no_change_outcome( - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - return await self._upsert_publications_outcome( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - - @staticmethod - def _apply_outcome_to_progress( - *, - progress: RunProgress, - run: CrawlRun, - outcome: ScholarProcessingOutcome, - ) -> None: - progress.succeeded_count += outcome.succeeded_count_delta - progress.failed_count += outcome.failed_count_delta - progress.partial_count += outcome.partial_count_delta - run.new_pub_count = int(run.new_pub_count or 0) + outcome.discovered_publication_count - progress.scholar_results.append(outcome.result_entry) - - def _unexpected_scholar_exception_outcome( - self, - *, - run: CrawlRun, - scholar: ScholarProfile, - start_cstart: int, - exc: Exception, - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = datetime.now(timezone.utc) - logger.exception( - "ingestion.scholar_unexpected_failure", - extra={ - "event": "ingestion.scholar_unexpected_failure", - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome( - result_entry={ - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": "ingestion_error", - "state_reason": "scholar_processing_exception", - "outcome": "failed", - "attempt_count": 0, - "publication_count": 0, - "start_cstart": start_cstart, - "warnings": [], - "error": str(exc), - "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, - }, - succeeded_count_delta=0, - failed_count_delta=1, - partial_count_delta=0, - discovered_publication_count=0, - ) - - @staticmethod - def _summarize_failures( - *, - scholar_results: list[dict[str, Any]], - ) -> RunFailureSummary: - failed_state_counts: dict[str, int] = {} - failed_reason_counts: dict[str, int] = {} - scrape_failure_counts: dict[str, int] = {} - retries_scheduled_count = 0 - scholars_with_retries_count = 0 - retry_exhausted_count = 0 - for entry in scholar_results: - retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1) - if retries_for_entry > 0: - retries_scheduled_count += retries_for_entry - scholars_with_retries_count += 1 - if str(entry.get("outcome", "")) != "failed": - continue - state = str(entry.get("state", "")).strip() - if state not in FAILED_STATES: - continue - failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 - reason = str(entry.get("state_reason", "")).strip() - if reason: - failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 - bucket = _classify_failure_bucket(state=state, state_reason=reason) - scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1 - if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0: - retry_exhausted_count += 1 - return RunFailureSummary( - failed_state_counts=failed_state_counts, - failed_reason_counts=failed_reason_counts, - scrape_failure_counts=scrape_failure_counts, - retries_scheduled_count=retries_scheduled_count, - scholars_with_retries_count=scholars_with_retries_count, - retry_exhausted_count=retry_exhausted_count, - ) - - @staticmethod - def _build_alert_summary( - *, - failure_summary: RunFailureSummary, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> RunAlertSummary: - blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) - network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) - blocked_threshold = max(1, int(alert_blocked_failure_threshold)) - network_threshold = max(1, int(alert_network_failure_threshold)) - retry_threshold = max(1, int(alert_retry_scheduled_threshold)) - alert_flags = { - "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold, - "network_failure_threshold_exceeded": network_failure_count >= network_threshold, - "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold, - } - return RunAlertSummary( - blocked_failure_count=blocked_failure_count, - network_failure_count=network_failure_count, - blocked_failure_threshold=blocked_threshold, - network_failure_threshold=network_threshold, - retry_scheduled_threshold=retry_threshold, - alert_flags=alert_flags, - ) - - @staticmethod - def _log_alert_thresholds( - *, - user_id: int, - run_id: int, - failure_summary: RunFailureSummary, - alert_summary: RunAlertSummary, - ) -> None: - if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]: - logger.warning( - "ingestion.alert_blocked_failure_threshold_exceeded", - extra={ - "event": "ingestion.alert_blocked_failure_threshold_exceeded", - "user_id": user_id, - "crawl_run_id": run_id, - "blocked_failure_count": alert_summary.blocked_failure_count, - "threshold": alert_summary.blocked_failure_threshold, - "metric_name": "ingestion_blocked_failure_threshold_exceeded_total", - "metric_value": 1, - }, - ) - if alert_summary.alert_flags["network_failure_threshold_exceeded"]: - logger.warning( - "ingestion.alert_network_failure_threshold_exceeded", - extra={ - "event": "ingestion.alert_network_failure_threshold_exceeded", - "user_id": user_id, - "crawl_run_id": run_id, - "network_failure_count": alert_summary.network_failure_count, - "threshold": alert_summary.network_failure_threshold, - "metric_name": "ingestion_network_failure_threshold_exceeded_total", - "metric_value": 1, - }, - ) - if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]: - logger.warning( - "ingestion.alert_retry_scheduled_threshold_exceeded", - extra={ - "event": "ingestion.alert_retry_scheduled_threshold_exceeded", - "user_id": user_id, - "crawl_run_id": run_id, - "retries_scheduled_count": failure_summary.retries_scheduled_count, - "threshold": alert_summary.retry_scheduled_threshold, - "metric_name": "ingestion_retry_scheduled_threshold_exceeded_total", - "metric_value": 1, - }, - ) - - @staticmethod - def _apply_safety_outcome( - *, - user_settings, - run: CrawlRun, - user_id: int, - alert_summary: RunAlertSummary, - ) -> None: - pre_apply_state = run_safety_service.get_safety_event_context( - user_settings, - now_utc=datetime.now(timezone.utc), - ) - safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome( - user_settings, - run_id=int(run.id), - blocked_failure_count=alert_summary.blocked_failure_count, - network_failure_count=alert_summary.network_failure_count, - blocked_failure_threshold=alert_summary.blocked_failure_threshold, - network_failure_threshold=alert_summary.network_failure_threshold, - blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, - network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, - now_utc=datetime.now(timezone.utc), - ) - ScholarIngestionService._log_safety_transition( - user_id=user_id, - run_id=int(run.id), - alert_summary=alert_summary, - pre_apply_state=pre_apply_state, - safety_state=safety_state, - cooldown_trigger_reason=cooldown_trigger_reason, - ) - - @staticmethod - def _log_safety_transition( - *, - user_id: int, - run_id: int, - alert_summary: RunAlertSummary, - pre_apply_state: dict[str, Any], - safety_state: dict[str, Any], - cooldown_trigger_reason: str | None, - ) -> None: - if cooldown_trigger_reason is not None: - logger.warning( - "ingestion.safety_cooldown_entered", - extra={ - "event": "ingestion.safety_cooldown_entered", - "user_id": user_id, - "crawl_run_id": run_id, - "reason": cooldown_trigger_reason, - "blocked_failure_count": alert_summary.blocked_failure_count, - "network_failure_count": alert_summary.network_failure_count, - "blocked_failure_threshold": alert_summary.blocked_failure_threshold, - "network_failure_threshold": alert_summary.network_failure_threshold, - "cooldown_until": safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": safety_state.get("cooldown_remaining_seconds"), - "safety_counters": safety_state.get("counters", {}), - "metric_name": "ingestion_safety_cooldown_entered_total", - "metric_value": 1, - }, - ) - elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"): - logger.info( - "ingestion.safety_cooldown_cleared", - extra={ - "event": "ingestion.safety_cooldown_cleared", - "user_id": user_id, - "crawl_run_id": run_id, - "reason": pre_apply_state.get("cooldown_reason"), - "cooldown_until": pre_apply_state.get("cooldown_until"), - "metric_name": "ingestion_safety_cooldown_cleared_total", - "metric_value": 1, - }, - ) - - def _finalize_run_record( - self, - *, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - failure_summary: RunFailureSummary, - alert_summary: RunAlertSummary, - idempotency_key: str | None, - ) -> None: - run.end_dt = datetime.now(timezone.utc) - run.status = self._resolve_run_status( - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - ) - run.error_log = { - "scholar_results": progress.scholar_results, - "summary": { - "succeeded_count": progress.succeeded_count, - "failed_count": progress.failed_count, - "partial_count": progress.partial_count, - "failed_state_counts": failure_summary.failed_state_counts, - "failed_reason_counts": failure_summary.failed_reason_counts, - "scrape_failure_counts": failure_summary.scrape_failure_counts, - "retry_counts": { - "retries_scheduled_count": failure_summary.retries_scheduled_count, - "scholars_with_retries_count": failure_summary.scholars_with_retries_count, - "retry_exhausted_count": failure_summary.retry_exhausted_count, - }, - "alert_thresholds": { - "blocked_failure_threshold": alert_summary.blocked_failure_threshold, - "network_failure_threshold": alert_summary.network_failure_threshold, - "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold, - }, - "alert_flags": alert_summary.alert_flags, - }, - "meta": {"idempotency_key": idempotency_key} if idempotency_key else {}, - } - - @staticmethod - def _log_run_completed( - *, - user_id: int, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - alert_summary: RunAlertSummary, - failure_summary: RunFailureSummary, - ) -> None: - logger.info( - "ingestion.run_completed", - extra={ - "event": "ingestion.run_completed", - "user_id": user_id, - "crawl_run_id": run.id, - "status": run.status.value, - "scholar_count": len(scholars), - "succeeded_count": progress.succeeded_count, - "failed_count": progress.failed_count, - "partial_count": progress.partial_count, - "new_publication_count": run.new_pub_count, - "blocked_failure_count": alert_summary.blocked_failure_count, - "network_failure_count": alert_summary.network_failure_count, - "retries_scheduled_count": failure_summary.retries_scheduled_count, - "alert_flags": alert_summary.alert_flags, - }, - ) - - @staticmethod - def _paging_kwargs( - *, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - ) -> dict[str, Any]: - return { - "request_delay_seconds": request_delay_seconds, - "network_error_retries": network_error_retries, - "retry_backoff_seconds": retry_backoff_seconds, - "max_pages_per_scholar": max_pages_per_scholar, - "page_size": page_size, - } - - @staticmethod - def _threshold_kwargs( - *, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> dict[str, Any]: - return { - "alert_blocked_failure_threshold": alert_blocked_failure_threshold, - "alert_network_failure_threshold": alert_network_failure_threshold, - "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, - } - - @staticmethod - def _run_execution_summary( - *, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - ) -> RunExecutionSummary: - return RunExecutionSummary( - crawl_run_id=run.id, - status=run.status, - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - new_publication_count=run.new_pub_count, - ) - - async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: - user_settings = await self._load_user_settings_for_run( - db_session, - user_id=user_id, - trigger_type=trigger_type, - ) - if not await self._try_acquire_user_lock(db_session, user_id=user_id): - raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") - filtered_scholar_ids, start_cstart_map = self._normalize_run_targets( - scholar_profile_ids=scholar_profile_ids, - start_cstart_by_scholar_id=start_cstart_by_scholar_id, - ) - scholars = await self._load_target_scholars( - db_session, - user_id=user_id, - filtered_scholar_ids=filtered_scholar_ids, - ) - run = await self._start_run_record_for_targets( - db_session, - user_id=user_id, - trigger_type=trigger_type, - scholars=scholars, - filtered=filtered_scholar_ids is not None, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - idempotency_key=idempotency_key, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - return user_settings, run, scholars, start_cstart_map - - async def _start_run_record_for_targets( - self, - db_session: AsyncSession, - *, - user_id: int, - trigger_type: RunTriggerType, - scholars: list[ScholarProfile], - filtered: bool, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> CrawlRun: - self._log_run_started( - user_id=user_id, - trigger_type=trigger_type, - scholar_count=len(scholars), - filtered=filtered, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - idempotency_key=idempotency_key, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - run = self._create_running_run( - user_id=user_id, - trigger_type=trigger_type, - scholar_count=len(scholars), - idempotency_key=idempotency_key, - ) - db_session.add(run) - await db_session.flush() - return run - - async def _run_scholar_iteration( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholars: list[ScholarProfile], - user_id: int, - start_cstart_map: dict[int, int], - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> RunProgress: - progress = RunProgress() - for index, scholar in enumerate(scholars): - await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) - start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) - outcome = await self._process_scholar( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - start_cstart=start_cstart, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - self._apply_outcome_to_progress(progress=progress, run=run, outcome=outcome) - return progress - - def _complete_run_for_user( - self, - *, - user_settings: Any, - run: CrawlRun, - scholars: list[ScholarProfile], - user_id: int, - progress: RunProgress, - idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> tuple[RunFailureSummary, RunAlertSummary]: - failure_summary = self._summarize_failures(scholar_results=progress.scholar_results) - alert_summary = self._build_alert_summary( - failure_summary=failure_summary, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - self._log_alert_thresholds( - user_id=user_id, - run_id=int(run.id), - failure_summary=failure_summary, - alert_summary=alert_summary, - ) - self._apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary) - self._finalize_run_record( - run=run, - scholars=scholars, - progress=progress, - failure_summary=failure_summary, - alert_summary=alert_summary, - idempotency_key=idempotency_key, - ) - return failure_summary, alert_summary - - async def run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, request_delay_seconds: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, max_pages_per_scholar: int = 30, page_size: int = 100, scholar_profile_ids: set[int] | None = None, start_cstart_by_scholar_id: dict[int, int] | None = None, auto_queue_continuations: bool = True, queue_delay_seconds: int = 60, idempotency_key: str | None = None, alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3) -> RunExecutionSummary: - paging_kwargs = self._paging_kwargs(request_delay_seconds=request_delay_seconds, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size) - threshold_kwargs = self._threshold_kwargs(alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold) - user_settings, run, scholars, start_cstart_map = await self._initialize_run_for_user( - db_session, - user_id=user_id, - trigger_type=trigger_type, - scholar_profile_ids=scholar_profile_ids, - start_cstart_by_scholar_id=start_cstart_by_scholar_id, - idempotency_key=idempotency_key, - **paging_kwargs, - **threshold_kwargs, - ) - progress = await self._run_scholar_iteration( - db_session, - run=run, - scholars=scholars, - user_id=user_id, - start_cstart_map=start_cstart_map, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - **paging_kwargs, - ) - failure_summary, alert_summary = self._complete_run_for_user( - user_settings=user_settings, - run=run, - scholars=scholars, - user_id=user_id, - progress=progress, - idempotency_key=idempotency_key, - **threshold_kwargs, - ) - await db_session.commit() - self._log_run_completed(user_id=user_id, run=run, scholars=scholars, progress=progress, alert_summary=alert_summary, failure_summary=failure_summary) - return self._run_execution_summary(run=run, scholars=scholars, progress=progress) - - async def _fetch_profile_page( - self, - *, - scholar_id: str, - cstart: int, - page_size: int, - ) -> FetchResult: - try: - page_fetcher = getattr(self._source, "fetch_profile_page_html", None) - if callable(page_fetcher): - return await page_fetcher( - scholar_id, - cstart=cstart, - pagesize=page_size, - ) - if cstart <= 0: - return await self._source.fetch_profile_html(scholar_id) - return FetchResult( - requested_url=( - "https://scholar.google.com/citations" - f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" - ), - status_code=None, - final_url=None, - body="", - error="source_does_not_support_pagination", - ) - except Exception as exc: - logger.exception( - "ingestion.fetch_unexpected_error", - extra={ - "event": "ingestion.fetch_unexpected_error", - "scholar_id": scholar_id, - "cstart": cstart, - "page_size": page_size, - }, - ) - return FetchResult( - requested_url=( - "https://scholar.google.com/citations" - f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}" - ), - status_code=None, - final_url=None, - body="", - error=str(exc), - ) - - @staticmethod - def _attempt_log_entry( - *, - attempt: int, - cstart: int, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - ) -> dict[str, Any]: - return { - "attempt": attempt, - "cstart": cstart, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "status_code": fetch_result.status_code, - "fetch_error": fetch_result.error, - } - - @staticmethod - def _should_retry_network_page( - *, - parsed_page: ParsedProfilePage, - attempt_index: int, - max_attempts: int, - ) -> bool: - return parsed_page.state == ParseState.NETWORK_ERROR and attempt_index < max_attempts - 1 - - @staticmethod - async def _sleep_retry_backoff( - *, - scholar_id: str, - cstart: int, - attempt_index: int, - backoff: float, - state_reason: str, - ) -> None: - sleep_seconds = backoff * (2**attempt_index) - logger.warning( - "ingestion.scholar_retry_scheduled", - extra={ - "event": "ingestion.scholar_retry_scheduled", - "scholar_id": scholar_id, - "cstart": cstart, - "attempt": attempt_index + 1, - "next_attempt": attempt_index + 2, - "sleep_seconds": sleep_seconds, - "state_reason": state_reason, - }, - ) - if sleep_seconds > 0: - await asyncio.sleep(sleep_seconds) - - async def _fetch_and_parse_page_with_retry( - self, - *, - scholar_id: str, - cstart: int, - page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: - max_attempts = max(1, int(network_error_retries) + 1) - backoff = max(float(retry_backoff_seconds), 0.0) - attempt_log: list[dict[str, Any]] = [] - fetch_result: FetchResult | None = None - parsed_page: ParsedProfilePage | None = None - - for attempt_index in range(max_attempts): - fetch_result = await self._fetch_profile_page( - scholar_id=scholar_id, - cstart=cstart, - page_size=page_size, - ) - parsed_page = parse_profile_page(fetch_result) - attempt_log.append( - self._attempt_log_entry( - attempt=attempt_index + 1, - cstart=cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - ) - ) - if not self._should_retry_network_page( - parsed_page=parsed_page, - attempt_index=attempt_index, - max_attempts=max_attempts, - ): - break - await self._sleep_retry_backoff( - scholar_id=scholar_id, - cstart=cstart, - attempt_index=attempt_index, - backoff=backoff, - state_reason=parsed_page.state_reason, - ) - - if fetch_result is None or parsed_page is None: - raise RuntimeError("Fetch-and-parse retry loop produced no result.") - return fetch_result, parsed_page, attempt_log - - @staticmethod - def _page_log_entry( - *, - page_number: int, - cstart: int, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - attempt_count: int, - ) -> dict[str, Any]: - return { - "page": page_number, - "cstart": cstart, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "status_code": fetch_result.status_code, - "publication_count": len(parsed_page.publications), - "articles_range": parsed_page.articles_range, - "has_show_more_button": parsed_page.has_show_more_button, - "warning_codes": parsed_page.warnings, - "attempt_count": attempt_count, - } - - @staticmethod - def _should_skip_no_change( - *, - start_cstart: int, - first_page_fingerprint_sha256: str | None, - previous_initial_page_fingerprint_sha256: str | None, - parsed_page: ParsedProfilePage, - ) -> bool: - return ( - start_cstart <= 0 - and first_page_fingerprint_sha256 is not None - and previous_initial_page_fingerprint_sha256 is not None - and first_page_fingerprint_sha256 == previous_initial_page_fingerprint_sha256 - and parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} - ) - - @staticmethod - def _skip_no_change_result( - *, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedParseResult: - return PagedParseResult( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fetch_result=fetch_result, - first_page_parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=[], - attempt_log=attempt_log, - page_logs=page_logs, - pages_fetched=1, - pages_attempted=1, - has_more_remaining=False, - pagination_truncated_reason=None, - continuation_cstart=None, - skipped_no_change=True, - ) - - @staticmethod - def _initial_failure_result( - *, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - start_cstart: int, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedParseResult: - continuation_cstart = start_cstart if parsed_page.state == ParseState.NETWORK_ERROR else None - return PagedParseResult( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fetch_result=fetch_result, - first_page_parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=[], - attempt_log=attempt_log, - page_logs=page_logs, - pages_fetched=0, - pages_attempted=1, - has_more_remaining=False, - pagination_truncated_reason=None, - continuation_cstart=continuation_cstart, - skipped_no_change=False, - ) - - @staticmethod - def _build_loop_state( - *, - start_cstart: int, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedLoopState: - next_cstart = _next_cstart_value( - articles_range=parsed_page.articles_range, - fallback=start_cstart + len(parsed_page.publications), - ) - return PagedLoopState( - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_log=attempt_log, - page_logs=page_logs, - publications=list(parsed_page.publications), - pages_fetched=1, - pages_attempted=1, - current_cstart=start_cstart, - next_cstart=next_cstart, - ) - - @staticmethod - def _set_truncated_state( - *, - state: PagedLoopState, - reason: str, - continuation_cstart: int, - ) -> None: - state.has_more_remaining = True - state.pagination_truncated_reason = reason - state.continuation_cstart = continuation_cstart - - def _should_stop_pagination(self, *, state: PagedLoopState, bounded_max_pages: int) -> bool: - if state.pages_fetched >= bounded_max_pages: - self._set_truncated_state( - state=state, - reason="max_pages_reached", - continuation_cstart=( - state.next_cstart if state.next_cstart > state.current_cstart else state.current_cstart - ), - ) - return True - if state.next_cstart <= state.current_cstart: - self._set_truncated_state( - state=state, - reason="pagination_cursor_stalled", - continuation_cstart=state.current_cstart, - ) - return True - return False - - async def _fetch_next_page( - self, - *, - scholar_id: str, - state: PagedLoopState, - request_delay_seconds: int, - bounded_page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - ) -> tuple[FetchResult, ParsedProfilePage, list[dict[str, Any]]]: - if request_delay_seconds > 0: - await asyncio.sleep(float(request_delay_seconds)) - state.current_cstart = state.next_cstart - return await self._fetch_and_parse_page_with_retry( - scholar_id=scholar_id, - cstart=state.current_cstart, - page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - - @staticmethod - def _record_next_page( - *, - state: PagedLoopState, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - page_attempt_log: list[dict[str, Any]], - ) -> None: - state.pages_attempted += 1 - state.attempt_log.extend(page_attempt_log) - state.page_logs.append( - ScholarIngestionService._page_log_entry( - page_number=state.pages_attempted, - cstart=state.current_cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_count=len(page_attempt_log), - ) - ) - state.fetch_result = fetch_result - state.parsed_page = parsed_page - - @staticmethod - def _handle_page_state_transition(*, state: PagedLoopState) -> bool: - if state.parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - ScholarIngestionService._set_truncated_state( - state=state, - reason=f"page_state_{state.parsed_page.state.value}", - continuation_cstart=state.current_cstart, - ) - return True - if state.parsed_page.state == ParseState.NO_RESULTS and len(state.parsed_page.publications) == 0: - state.pages_fetched += 1 - return True - state.pages_fetched += 1 - state.publications.extend(state.parsed_page.publications) - state.next_cstart = _next_cstart_value( - articles_range=state.parsed_page.articles_range, - fallback=state.current_cstart + len(state.parsed_page.publications), - ) - return False - - async def _fetch_initial_page_context( - self, - *, - scholar_id: str, - start_cstart: int, - bounded_page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - ) -> tuple[FetchResult, ParsedProfilePage, str | None, list[dict[str, Any]], list[dict[str, Any]]]: - fetch_result, parsed_page, first_attempt_log = await self._fetch_and_parse_page_with_retry( - scholar_id=scholar_id, - cstart=start_cstart, - page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page) - attempt_log = list(first_attempt_log) - page_logs = [ - self._page_log_entry( - page_number=1, - cstart=start_cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_count=len(first_attempt_log), - ) - ] - return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs - - async def _paginate_loop( - self, - *, - scholar_id: str, - state: PagedLoopState, - bounded_max_pages: int, - request_delay_seconds: int, - bounded_page_size: int, - network_error_retries: int, - retry_backoff_seconds: float, - ) -> None: - while state.parsed_page.has_show_more_button: - if self._should_stop_pagination(state=state, bounded_max_pages=bounded_max_pages): - return - next_fetch_result, next_parsed_page, next_attempt_log = await self._fetch_next_page( - scholar_id=scholar_id, - state=state, - request_delay_seconds=request_delay_seconds, - bounded_page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - self._record_next_page( - state=state, - fetch_result=next_fetch_result, - parsed_page=next_parsed_page, - page_attempt_log=next_attempt_log, - ) - if self._handle_page_state_transition(state=state): - return - - @staticmethod - def _result_from_pagination_state( - *, - state: PagedLoopState, - first_page_fetch_result: FetchResult, - first_page_parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - ) -> PagedParseResult: - return PagedParseResult( - fetch_result=state.fetch_result, - parsed_page=state.parsed_page, - first_page_fetch_result=first_page_fetch_result, - first_page_parsed_page=first_page_parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - publications=_dedupe_publication_candidates(state.publications), - attempt_log=state.attempt_log, - page_logs=state.page_logs, - pages_fetched=state.pages_fetched, - pages_attempted=state.pages_attempted, - has_more_remaining=state.has_more_remaining, - pagination_truncated_reason=state.pagination_truncated_reason, - continuation_cstart=state.continuation_cstart, - skipped_no_change=False, - ) - - def _short_circuit_initial_page( - self, - *, - start_cstart: int, - previous_initial_page_fingerprint_sha256: str | None, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - first_page_fingerprint_sha256: str | None, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]], - ) -> PagedParseResult | None: - if self._should_skip_no_change( - start_cstart=start_cstart, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, - parsed_page=parsed_page, - ): - return self._skip_no_change_result( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - attempt_log=attempt_log, - page_logs=page_logs, - ) - if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - return self._initial_failure_result( - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - start_cstart=start_cstart, - attempt_log=attempt_log, - page_logs=page_logs, - ) - return None - - async def _fetch_and_parse_all_pages_with_retry(self, *, scholar_id: str, start_cstart: int, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, max_pages: int, page_size: int, previous_initial_page_fingerprint_sha256: str | None = None) -> PagedParseResult: - bounded_max_pages = max(1, int(max_pages)) - bounded_page_size = max(1, int(page_size)) - fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs = ( - await self._fetch_initial_page_context( - scholar_id=scholar_id, - start_cstart=start_cstart, - bounded_page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - )) - shortcut_result = self._short_circuit_initial_page( - start_cstart=start_cstart, - previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256, - fetch_result=fetch_result, - parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - attempt_log=attempt_log, - page_logs=page_logs, - ) - if shortcut_result is not None: - return shortcut_result - state = self._build_loop_state( - start_cstart=start_cstart, - fetch_result=fetch_result, - parsed_page=parsed_page, - attempt_log=attempt_log, - page_logs=page_logs, - ) - await self._paginate_loop( - scholar_id=scholar_id, - state=state, - bounded_max_pages=bounded_max_pages, - request_delay_seconds=request_delay_seconds, - bounded_page_size=bounded_page_size, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - return self._result_from_pagination_state( - state=state, - first_page_fetch_result=fetch_result, - first_page_parsed_page=parsed_page, - first_page_fingerprint_sha256=first_page_fingerprint_sha256, - ) - - async def _upsert_profile_publications( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - publications: list[PublicationCandidate], - ) -> int: - seen_publication_ids: set[int] = set() - discovered_count = 0 - - for candidate in publications: - publication = await self._resolve_publication(db_session, candidate) - if publication.id in seen_publication_ids: - continue - seen_publication_ids.add(publication.id) - - link_result = await db_session.execute( - select(ScholarPublication).where( - ScholarPublication.scholar_profile_id == scholar.id, - ScholarPublication.publication_id == publication.id, - ) - ) - link = link_result.scalar_one_or_none() - if link is not None: - continue - - link = ScholarPublication( - scholar_profile_id=scholar.id, - publication_id=publication.id, - is_read=False, - first_seen_run_id=run.id, - ) - db_session.add(link) - discovered_count += 1 - - logger.debug( - "ingestion.publication_discovered", - extra={ - "event": "ingestion.publication_discovered", - "scholar_profile_id": scholar.id, - "publication_id": publication.id, - "crawl_run_id": run.id, - }, - ) - - if not scholar.baseline_completed: - scholar.baseline_completed = True - - return discovered_count - - @staticmethod - def _validate_publication_candidate(candidate: PublicationCandidate) -> None: - if not candidate.title.strip(): - raise RuntimeError("Publication candidate is missing title.") - if candidate.citation_count is not None and int(candidate.citation_count) < 0: - raise RuntimeError("Publication candidate has negative citation_count.") - - async def _find_publication_by_cluster( - self, - db_session: AsyncSession, - *, - cluster_id: str | None, - ) -> Publication | None: - if not cluster_id: - return None - result = await db_session.execute( - select(Publication).where(Publication.cluster_id == cluster_id) - ) - return result.scalar_one_or_none() - - async def _find_publication_by_fingerprint( - self, - db_session: AsyncSession, - *, - fingerprint: str, - ) -> Publication | None: - result = await db_session.execute( - select(Publication).where(Publication.fingerprint_sha256 == fingerprint) - ) - return result.scalar_one_or_none() - - @staticmethod - def _select_existing_publication( - *, - cluster_publication: Publication | None, - fingerprint_publication: Publication | None, - ) -> Publication | None: - if cluster_publication is not None: - return cluster_publication - return fingerprint_publication - - async def _create_publication( - self, - db_session: AsyncSession, - *, - candidate: PublicationCandidate, - fingerprint: str, - ) -> Publication: - publication = Publication( - cluster_id=candidate.cluster_id, - fingerprint_sha256=fingerprint, - title_raw=candidate.title, - title_normalized=normalize_title(candidate.title), - year=candidate.year, - citation_count=int(candidate.citation_count or 0), - author_text=candidate.authors_text, - venue_text=candidate.venue_text, - pub_url=build_publication_url(candidate.title_url), - pdf_url=build_publication_url(candidate.pdf_url), - ) - db_session.add(publication) - await db_session.flush() - logger.debug( - "ingestion.publication_created", - extra={ - "event": "ingestion.publication_created", - "publication_id": publication.id, - "cluster_id": publication.cluster_id, - }, - ) - return publication - - @staticmethod - def _update_existing_publication( - *, - publication: Publication, - candidate: PublicationCandidate, - ) -> None: - if candidate.cluster_id and publication.cluster_id is None: - publication.cluster_id = candidate.cluster_id - publication.title_raw = candidate.title - publication.title_normalized = normalize_title(candidate.title) - if candidate.year is not None: - publication.year = candidate.year - if candidate.citation_count is not None: - publication.citation_count = int(candidate.citation_count) - if candidate.authors_text: - publication.author_text = candidate.authors_text - if candidate.venue_text: - publication.venue_text = candidate.venue_text - if candidate.title_url: - publication.pub_url = build_publication_url(candidate.title_url) - if candidate.pdf_url: - publication.pdf_url = build_publication_url(candidate.pdf_url) - - async def _resolve_publication( - self, - db_session: AsyncSession, - candidate: PublicationCandidate, - ) -> Publication: - self._validate_publication_candidate(candidate) - fingerprint = build_publication_fingerprint(candidate) - cluster_publication = await self._find_publication_by_cluster( - db_session, - cluster_id=candidate.cluster_id, - ) - fingerprint_publication = await self._find_publication_by_fingerprint( - db_session, - fingerprint=fingerprint, - ) - publication = self._select_existing_publication( - cluster_publication=cluster_publication, - fingerprint_publication=fingerprint_publication, - ) - if publication is None: - return await self._create_publication( - db_session, - candidate=candidate, - fingerprint=fingerprint, - ) - self._update_existing_publication( - publication=publication, - candidate=candidate, - ) - return publication - - def _resolve_run_status( - self, - *, - scholar_count: int, - succeeded_count: int, - failed_count: int, - partial_count: int, - ) -> RunStatus: - if scholar_count == 0: - return RunStatus.SUCCESS - if failed_count == scholar_count: - return RunStatus.FAILED - if failed_count > 0 or partial_count > 0: - return RunStatus.PARTIAL_FAILURE - if succeeded_count > 0: - return RunStatus.SUCCESS - return RunStatus.FAILED - - def _resolve_continuation_queue_target( - self, - *, - outcome: str, - state: str, - pagination_truncated_reason: str | None, - continuation_cstart: int | None, - fallback_cstart: int, - ) -> tuple[str | None, int]: - if outcome == "partial": - reason = (pagination_truncated_reason or "").strip() - if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith( - RESUMABLE_PARTIAL_REASON_PREFIXES - ): - return reason, queue_service.normalize_cstart( - continuation_cstart if continuation_cstart is not None else fallback_cstart - ) - return None, queue_service.normalize_cstart(fallback_cstart) - - if outcome == "failed" and state == ParseState.NETWORK_ERROR.value: - return "network_error_retry", queue_service.normalize_cstart( - continuation_cstart if continuation_cstart is not None else fallback_cstart - ) - - return None, queue_service.normalize_cstart(fallback_cstart) - - def _build_failure_debug_context( - self, - *, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]] | None = None, - exception: Exception | None = None, - ) -> dict[str, Any]: - context: dict[str, Any] = { - "requested_url": fetch_result.requested_url, - "final_url": fetch_result.final_url, - "status_code": fetch_result.status_code, - "fetch_error": fetch_result.error, - "state_reason": parsed_page.state_reason, - "profile_name": parsed_page.profile_name, - "profile_image_url": parsed_page.profile_image_url, - "articles_range": parsed_page.articles_range, - "has_show_more_button": parsed_page.has_show_more_button, - "has_operation_error_banner": parsed_page.has_operation_error_banner, - "warning_codes": parsed_page.warnings, - "marker_counts_nonzero": { - key: value for key, value in parsed_page.marker_counts.items() if value > 0 - }, - "body_length": len(fetch_result.body), - "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() - if fetch_result.body - else None, - "body_excerpt": _build_body_excerpt(fetch_result.body), - "attempt_log": attempt_log, - } - if page_logs: - context["page_logs"] = page_logs - if exception is not None: - context["exception_type"] = type(exception).__name__ - context["exception_message"] = str(exception) - return context - - async def _try_acquire_user_lock( - self, - db_session: AsyncSession, - *, - user_id: int, - ) -> bool: - result = await db_session.execute( - text( - "SELECT pg_try_advisory_xact_lock(:namespace, :user_key)" - ), - { - "namespace": RUN_LOCK_NAMESPACE, - "user_key": int(user_id), - }, - ) - return bool(result.scalar_one()) - - -def normalize_title(value: str) -> str: - lowered = value.lower() - cleaned = TITLE_ALNUM_RE.sub("", lowered) - return cleaned - - -def _first_author_last_name(authors_text: str | None) -> str: - if not authors_text: - return "" - first_author = authors_text.split(",", maxsplit=1)[0].strip().lower() - words = WORD_RE.findall(first_author) - if not words: - return "" - return words[-1] - - -def _first_venue_word(venue_text: str | None) -> str: - if not venue_text: - return "" - words = WORD_RE.findall(venue_text.lower()) - if not words: - return "" - return words[0] - - -def build_publication_fingerprint(candidate: PublicationCandidate) -> str: - canonical = "|".join( - [ - normalize_title(candidate.title), - str(candidate.year) if candidate.year is not None else "", - _first_author_last_name(candidate.authors_text), - _first_venue_word(candidate.venue_text), - ] - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def build_initial_page_fingerprint(parsed_page: ParsedProfilePage) -> str | None: - if parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}: - return None - - normalized_rows: list[dict[str, Any]] = [] - for publication in parsed_page.publications[:INITIAL_PAGE_FINGERPRINT_MAX_PUBLICATIONS]: - normalized_rows.append( - { - "cluster_id": publication.cluster_id or "", - "title_normalized": normalize_title(publication.title), - "year": publication.year, - "citation_count": publication.citation_count, - } - ) - - payload = { - "state": parsed_page.state.value, - "articles_range": parsed_page.articles_range or "", - "has_show_more_button": parsed_page.has_show_more_button, - "profile_name": parsed_page.profile_name or "", - "publications": normalized_rows, - } - canonical = json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=True, - ) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() - - -def build_publication_url(path_or_url: str | None) -> str | None: - if not path_or_url: - return None - return urljoin("https://scholar.google.com", path_or_url) - - -def _next_cstart_value(*, articles_range: str | None, fallback: int) -> int: - if articles_range: - numbers = re.findall(r"\d+", articles_range) - if len(numbers) >= 2: - try: - return int(numbers[1]) - except ValueError: - pass - return int(fallback) - - -def _dedupe_publication_candidates( - publications: list[PublicationCandidate], -) -> list[PublicationCandidate]: - deduped: list[PublicationCandidate] = [] - seen: set[str] = set() - for publication in publications: - if publication.cluster_id: - identity = f"cluster:{publication.cluster_id}" - else: - identity = "|".join( - [ - "fallback", - normalize_title(publication.title), - str(publication.year) if publication.year is not None else "", - publication.authors_text or "", - publication.venue_text or "", - ] - ) - if identity in seen: - continue - seen.add(identity) - deduped.append(publication) - return deduped - - -def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None: - if not body: - return None - flattened = SPACE_RE.sub(" ", HTML_TAG_RE.sub(" ", body)).strip() - if not flattened: - return None - if len(flattened) <= max_chars: - return flattened - return f"{flattened[:max_chars - 1]}..." +from app.services.domains.ingestion.application import * diff --git a/app/services/publications.py b/app/services/publications.py index 4f2948f..f77b641 100644 --- a/app/services/publications.py +++ b/app/services/publications.py @@ -1,385 +1,3 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import datetime - -from sqlalchemy import Select, func, select, tuple_, update -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ( - CrawlRun, - Publication, - RunStatus, - ScholarProfile, - ScholarPublication, -) - -MODE_ALL = "all" -MODE_UNREAD = "unread" -MODE_LATEST = "latest" -MODE_NEW = "new" # compatibility alias for MODE_LATEST - - -@dataclass(frozen=True) -class PublicationListItem: - publication_id: int - scholar_profile_id: int - scholar_label: str - title: str - year: int | None - citation_count: int - venue_text: str | None - pub_url: str | None - pdf_url: str | None - is_read: bool - first_seen_at: datetime - is_new_in_latest_run: bool - - -@dataclass(frozen=True) -class UnreadPublicationItem: - publication_id: int - scholar_profile_id: int - scholar_label: str - title: str - year: int | None - citation_count: int - venue_text: str | None - pub_url: str | None - pdf_url: str | None - - -def resolve_publication_view_mode(value: str | None) -> str: - if value == MODE_UNREAD: - return MODE_UNREAD - if value in {MODE_LATEST, MODE_NEW}: - return MODE_LATEST - return MODE_ALL - - -def resolve_mode(value: str | None) -> str: - return resolve_publication_view_mode(value) - - -async def get_latest_completed_run_id_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> int | None: - result = await db_session.execute( - select(func.max(CrawlRun.id)).where( - CrawlRun.user_id == user_id, - CrawlRun.status != RunStatus.RUNNING, - ) - ) - latest_run_id = result.scalar_one_or_none() - return int(latest_run_id) if latest_run_id is not None else None - - -def publications_query( - *, - user_id: int, - mode: str, - latest_run_id: int | None, - scholar_profile_id: int | None, - limit: int, -) -> Select[tuple]: - scholar_label = ScholarProfile.display_name - - stmt = ( - select( - Publication.id, - ScholarProfile.id, - scholar_label, - ScholarProfile.scholar_id, - Publication.title_raw, - Publication.year, - Publication.citation_count, - Publication.venue_text, - Publication.pub_url, - Publication.pdf_url, - ScholarPublication.is_read, - ScholarPublication.first_seen_run_id, - ScholarPublication.created_at, - ) - .join(ScholarPublication, ScholarPublication.publication_id == Publication.id) - .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarPublication.created_at.desc(), Publication.id.desc()) - .limit(limit) - ) - if scholar_profile_id is not None: - stmt = stmt.where(ScholarProfile.id == scholar_profile_id) - if mode == MODE_UNREAD: - stmt = stmt.where(ScholarPublication.is_read.is_(False)) - if mode == MODE_LATEST: - # "Latest" means discovered in the latest completed run. - if latest_run_id is None: - stmt = stmt.where(False) - else: - stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) - return stmt - - -def _publication_list_item_from_row( - row: tuple, - *, - latest_run_id: int | None, -) -> PublicationListItem: - ( - publication_id, - scholar_profile_id, - display_name, - scholar_id, - title_raw, - year, - citation_count, - venue_text, - pub_url, - pdf_url, - is_read, - first_seen_run_id, - created_at, - ) = row - return PublicationListItem( - publication_id=int(publication_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - title=title_raw, - year=year, - citation_count=int(citation_count or 0), - venue_text=venue_text, - pub_url=pub_url, - pdf_url=pdf_url, - is_read=bool(is_read), - first_seen_at=created_at, - is_new_in_latest_run=( - latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id - ), - ) - - -async def list_for_user( - db_session: AsyncSession, - *, - user_id: int, - mode: str = MODE_ALL, - scholar_profile_id: int | None = None, - limit: int = 300, -) -> list[PublicationListItem]: - resolved_mode = resolve_publication_view_mode(mode) - latest_run_id = await get_latest_completed_run_id_for_user( - db_session, - user_id=user_id, - ) - result = await db_session.execute( - publications_query( - user_id=user_id, - mode=resolved_mode, - latest_run_id=latest_run_id, - scholar_profile_id=scholar_profile_id, - limit=limit, - ) - ) - return [ - _publication_list_item_from_row(row, latest_run_id=latest_run_id) - for row in result.all() - ] - - -async def list_unread_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 100, -) -> list[UnreadPublicationItem]: - result = await db_session.execute( - publications_query( - user_id=user_id, - mode=MODE_UNREAD, - latest_run_id=None, - scholar_profile_id=None, - limit=limit, - ) - ) - rows = result.all() - items: list[UnreadPublicationItem] = [] - for row in rows: - ( - publication_id, - scholar_profile_id, - display_name, - scholar_id, - title_raw, - year, - citation_count, - venue_text, - pub_url, - pdf_url, - _is_read, - _first_seen_run_id, - _created_at, - ) = row - items.append( - UnreadPublicationItem( - publication_id=int(publication_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - title=title_raw, - year=year, - citation_count=int(citation_count or 0), - venue_text=venue_text, - pub_url=pub_url, - pdf_url=pdf_url, - ) - ) - return items - - -async def list_new_for_latest_run_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 100, -) -> list[UnreadPublicationItem]: - rows = await list_for_user( - db_session, - user_id=user_id, - mode=MODE_LATEST, - scholar_profile_id=None, - limit=limit, - ) - return [ - UnreadPublicationItem( - publication_id=row.publication_id, - scholar_profile_id=row.scholar_profile_id, - scholar_label=row.scholar_label, - title=row.title, - year=row.year, - citation_count=row.citation_count, - venue_text=row.venue_text, - pub_url=row.pub_url, - pdf_url=row.pdf_url, - ) - for row in rows - ] - - -async def count_for_user( - db_session: AsyncSession, - *, - user_id: int, - mode: str = MODE_ALL, - scholar_profile_id: int | None = None, -) -> int: - resolved_mode = resolve_publication_view_mode(mode) - latest_run_id = await get_latest_completed_run_id_for_user( - db_session, - user_id=user_id, - ) - stmt = ( - select(func.count()) - .select_from(ScholarPublication) - .join(ScholarProfile, ScholarProfile.id == ScholarPublication.scholar_profile_id) - .where(ScholarProfile.user_id == user_id) - ) - if scholar_profile_id is not None: - stmt = stmt.where(ScholarProfile.id == scholar_profile_id) - if resolved_mode == MODE_UNREAD: - stmt = stmt.where(ScholarPublication.is_read.is_(False)) - if resolved_mode == MODE_LATEST: - if latest_run_id is None: - return 0 - stmt = stmt.where(ScholarPublication.first_seen_run_id == latest_run_id) - result = await db_session.execute(stmt) - return int(result.scalar_one() or 0) - - -async def count_unread_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int | None = None, -) -> int: - return await count_for_user( - db_session, - user_id=user_id, - mode=MODE_UNREAD, - scholar_profile_id=scholar_profile_id, - ) - - -async def count_latest_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int | None = None, -) -> int: - return await count_for_user( - db_session, - user_id=user_id, - mode=MODE_LATEST, - scholar_profile_id=scholar_profile_id, - ) - - -async def mark_all_unread_as_read_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> int: - scholar_ids = ( - select(ScholarProfile.id) - .where(ScholarProfile.user_id == user_id) - .scalar_subquery() - ) - - stmt = ( - update(ScholarPublication) - .where( - ScholarPublication.scholar_profile_id.in_(scholar_ids), - ScholarPublication.is_read.is_(False), - ) - .values(is_read=True) - ) - result = await db_session.execute(stmt) - await db_session.commit() - - rowcount = result.rowcount - return int(rowcount or 0) - - -async def mark_selected_as_read_for_user( - db_session: AsyncSession, - *, - user_id: int, - selections: list[tuple[int, int]], -) -> int: - normalized_pairs = { - (int(scholar_profile_id), int(publication_id)) - for scholar_profile_id, publication_id in selections - if int(scholar_profile_id) > 0 and int(publication_id) > 0 - } - if not normalized_pairs: - return 0 - - scholar_ids = ( - select(ScholarProfile.id) - .where(ScholarProfile.user_id == user_id) - .scalar_subquery() - ) - stmt = ( - update(ScholarPublication) - .where( - ScholarPublication.scholar_profile_id.in_(scholar_ids), - tuple_( - ScholarPublication.scholar_profile_id, - ScholarPublication.publication_id, - ).in_(list(normalized_pairs)), - ScholarPublication.is_read.is_(False), - ) - .values(is_read=True) - ) - result = await db_session.execute(stmt) - await db_session.commit() - return int(result.rowcount or 0) +from app.services.domains.publications.application import * diff --git a/app/services/run_safety.py b/app/services/run_safety.py index debb8c5..1c9329d 100644 --- a/app/services/run_safety.py +++ b/app/services/run_safety.py @@ -1,282 +1,3 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone -from typing import Any - -from app.db.models import UserSetting - -COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD = "blocked_failure_threshold_exceeded" -COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD = "network_failure_threshold_exceeded" - -_COUNTER_CONSECUTIVE_BLOCKED_RUNS = "consecutive_blocked_runs" -_COUNTER_CONSECUTIVE_NETWORK_RUNS = "consecutive_network_runs" -_COUNTER_COOLDOWN_ENTRY_COUNT = "cooldown_entry_count" -_COUNTER_BLOCKED_START_COUNT = "blocked_start_count" -_COUNTER_LAST_BLOCKED_FAILURE_COUNT = "last_blocked_failure_count" -_COUNTER_LAST_NETWORK_FAILURE_COUNT = "last_network_failure_count" -_COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id" - - -def _utcnow() -> datetime: - return datetime.now(timezone.utc) - - -def _safe_int(value: Any, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _safe_optional_int(value: Any) -> int | None: - if value is None: - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _normalize_datetime(value: datetime | None) -> datetime | None: - if value is None: - return None - if value.tzinfo is None: - return value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc) - - -def _state_dict(settings: UserSetting) -> dict[str, Any]: - state = settings.scrape_safety_state - if isinstance(state, dict): - return state - return {} - - -def _counters_from_state(settings: UserSetting) -> dict[str, Any]: - state = _state_dict(settings) - return { - _COUNTER_CONSECUTIVE_BLOCKED_RUNS: max( - 0, - _safe_int(state.get(_COUNTER_CONSECUTIVE_BLOCKED_RUNS), 0), - ), - _COUNTER_CONSECUTIVE_NETWORK_RUNS: max( - 0, - _safe_int(state.get(_COUNTER_CONSECUTIVE_NETWORK_RUNS), 0), - ), - _COUNTER_COOLDOWN_ENTRY_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_COOLDOWN_ENTRY_COUNT), 0), - ), - _COUNTER_BLOCKED_START_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_BLOCKED_START_COUNT), 0), - ), - _COUNTER_LAST_BLOCKED_FAILURE_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_LAST_BLOCKED_FAILURE_COUNT), 0), - ), - _COUNTER_LAST_NETWORK_FAILURE_COUNT: max( - 0, - _safe_int(state.get(_COUNTER_LAST_NETWORK_FAILURE_COUNT), 0), - ), - _COUNTER_LAST_EVALUATED_RUN_ID: _safe_optional_int( - state.get(_COUNTER_LAST_EVALUATED_RUN_ID), - ), - } - - -def _cooldown_reason_label(reason: str | None) -> str | None: - if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: - return "Blocked responses exceeded safety threshold" - if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: - return "Network failures exceeded safety threshold" - return None - - -def _recommended_action(reason: str | None) -> str | None: - if reason == COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD: - return ( - "Google Scholar appears to be blocking requests. Wait for cooldown to expire, " - "increase request delay, and avoid repeated manual retries." - ) - if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD: - return ( - "Network failures crossed the threshold. Verify connectivity and retry after cooldown." - ) - return None - - -def is_cooldown_active( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> bool: - now = now_utc or _utcnow() - cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) - if cooldown_until is None: - return False - return cooldown_until > now - - -def clear_expired_cooldown( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> bool: - now = now_utc or _utcnow() - cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) - if cooldown_until is None: - return False - if cooldown_until > now: - return False - settings.scrape_cooldown_until = None - settings.scrape_cooldown_reason = None - return True - - -def register_cooldown_blocked_start( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> dict[str, Any]: - now = now_utc or _utcnow() - counters = _counters_from_state(settings) - counters[_COUNTER_BLOCKED_START_COUNT] = int(counters[_COUNTER_BLOCKED_START_COUNT]) + 1 - settings.scrape_safety_state = counters - return get_safety_state_payload(settings, now_utc=now) - - -def _update_run_counters( - *, - counters: dict[str, Any], - run_id: int, - blocked_failure_count: int, - network_failure_count: int, -) -> tuple[int, int]: - bounded_blocked_failures = max(0, int(blocked_failure_count)) - bounded_network_failures = max(0, int(network_failure_count)) - counters[_COUNTER_LAST_BLOCKED_FAILURE_COUNT] = bounded_blocked_failures - counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures - counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id) - counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = ( - int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 - if bounded_blocked_failures > 0 - else 0 - ) - counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = ( - int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 - if bounded_network_failures > 0 - else 0 - ) - return bounded_blocked_failures, bounded_network_failures - - -def _resolve_cooldown_trigger( - *, - blocked_failures: int, - network_failures: int, - blocked_failure_threshold: int, - network_failure_threshold: int, - blocked_cooldown_seconds: int, - network_cooldown_seconds: int, -) -> tuple[str | None, int]: - if blocked_failures >= max(1, int(blocked_failure_threshold)): - return COOLDOWN_REASON_BLOCKED_FAILURE_THRESHOLD, max(60, int(blocked_cooldown_seconds)) - if network_failures >= max(1, int(network_failure_threshold)): - return COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD, max(60, int(network_cooldown_seconds)) - return None, 0 - - -def _apply_cooldown_decision( - *, - settings: UserSetting, - counters: dict[str, Any], - now: datetime, - reason: str | None, - cooldown_seconds: int, -) -> None: - if reason is None: - clear_expired_cooldown(settings, now_utc=now) - return - settings.scrape_cooldown_reason = reason - settings.scrape_cooldown_until = now + timedelta(seconds=max(60, int(cooldown_seconds))) - counters[_COUNTER_COOLDOWN_ENTRY_COUNT] = int(counters[_COUNTER_COOLDOWN_ENTRY_COUNT]) + 1 - - -def apply_run_safety_outcome( - settings: UserSetting, - *, - run_id: int, - blocked_failure_count: int, - network_failure_count: int, - blocked_failure_threshold: int, - network_failure_threshold: int, - blocked_cooldown_seconds: int, - network_cooldown_seconds: int, - now_utc: datetime | None = None, -) -> tuple[dict[str, Any], str | None]: - now = now_utc or _utcnow() - counters = _counters_from_state(settings) - blocked_failures, network_failures = _update_run_counters( - counters=counters, - run_id=run_id, - blocked_failure_count=blocked_failure_count, - network_failure_count=network_failure_count, - ) - reason, cooldown_seconds = _resolve_cooldown_trigger( - blocked_failures=blocked_failures, - network_failures=network_failures, - blocked_failure_threshold=blocked_failure_threshold, - network_failure_threshold=network_failure_threshold, - blocked_cooldown_seconds=blocked_cooldown_seconds, - network_cooldown_seconds=network_cooldown_seconds, - ) - _apply_cooldown_decision( - settings=settings, - counters=counters, - now=now, - reason=reason, - cooldown_seconds=cooldown_seconds, - ) - settings.scrape_safety_state = counters - return get_safety_state_payload(settings, now_utc=now), reason - - -def get_safety_state_payload( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> dict[str, Any]: - now = now_utc or _utcnow() - cooldown_until = _normalize_datetime(settings.scrape_cooldown_until) - cooldown_active = bool(cooldown_until is not None and cooldown_until > now) - cooldown_remaining_seconds = 0 - if cooldown_active and cooldown_until is not None: - cooldown_remaining_seconds = max(0, int((cooldown_until - now).total_seconds())) - - reason = settings.scrape_cooldown_reason if cooldown_active else None - - return { - "cooldown_active": cooldown_active, - "cooldown_reason": reason, - "cooldown_reason_label": _cooldown_reason_label(reason), - "cooldown_until": cooldown_until, - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "recommended_action": _recommended_action(reason), - "counters": _counters_from_state(settings), - } - - -def get_safety_event_context( - settings: UserSetting, - *, - now_utc: datetime | None = None, -) -> dict[str, Any]: - payload = get_safety_state_payload(settings, now_utc=now_utc) - return { - "cooldown_active": bool(payload.get("cooldown_active")), - "cooldown_reason": payload.get("cooldown_reason"), - "cooldown_until": payload.get("cooldown_until"), - "cooldown_remaining_seconds": int(payload.get("cooldown_remaining_seconds") or 0), - "safety_counters": payload.get("counters", {}), - } +from app.services.domains.ingestion.safety import * diff --git a/app/services/runs.py b/app/services/runs.py index 3d20ac4..e917e47 100644 --- a/app/services/runs.py +++ b/app/services/runs.py @@ -1,435 +1,3 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import datetime -from typing import Any - -from sqlalchemy import and_, case, func, or_, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ( - CrawlRun, - IngestionQueueItem, - RunStatus, - RunTriggerType, - ScholarProfile, -) -from app.services import continuation_queue as queue_service - -QUEUE_STATUS_QUEUED = "queued" -QUEUE_STATUS_RETRYING = "retrying" -QUEUE_STATUS_DROPPED = "dropped" - - -@dataclass(frozen=True) -class QueueListItem: - id: int - scholar_profile_id: int - scholar_label: str - status: str - reason: str - dropped_reason: str | None - attempt_count: int - resume_cstart: int - next_attempt_dt: datetime | None - updated_at: datetime - last_error: str | None - last_run_id: int | None - - -@dataclass(frozen=True) -class QueueClearResult: - queue_item_id: int - previous_status: str - - -class QueueTransitionError(RuntimeError): - def __init__(self, *, code: str, message: str, current_status: str) -> None: - super().__init__(message) - self.code = code - self.message = message - self.current_status = current_status - - -def _safe_int(value: object, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _summary_dict(error_log: object) -> dict[str, Any]: - if not isinstance(error_log, dict): - return {} - summary = error_log.get("summary") - if not isinstance(summary, dict): - return {} - return summary - - -def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]: - value = summary.get(key) - if not isinstance(value, dict): - return {} - return { - str(item_key): _safe_int(item_value, 0) - for item_key, item_value in value.items() - if isinstance(item_key, str) - } - - -def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]: - value = summary.get(key) - if not isinstance(value, dict): - return {} - return { - str(item_key): bool(item_value) - for item_key, item_value in value.items() - if isinstance(item_key, str) - } - - -def extract_run_summary(error_log: object) -> dict[str, Any]: - summary = _summary_dict(error_log) - return { - "succeeded_count": _safe_int(summary.get("succeeded_count", 0)), - "failed_count": _safe_int(summary.get("failed_count", 0)), - "partial_count": _safe_int(summary.get("partial_count", 0)), - "failed_state_counts": _summary_int_dict(summary, "failed_state_counts"), - "failed_reason_counts": _summary_int_dict(summary, "failed_reason_counts"), - "scrape_failure_counts": _summary_int_dict(summary, "scrape_failure_counts"), - "retry_counts": { - "retries_scheduled_count": _safe_int( - ( - summary.get("retry_counts", {}).get("retries_scheduled_count") - if isinstance(summary.get("retry_counts"), dict) - else 0 - ), - 0, - ), - "scholars_with_retries_count": _safe_int( - ( - summary.get("retry_counts", {}).get("scholars_with_retries_count") - if isinstance(summary.get("retry_counts"), dict) - else 0 - ), - 0, - ), - "retry_exhausted_count": _safe_int( - ( - summary.get("retry_counts", {}).get("retry_exhausted_count") - if isinstance(summary.get("retry_counts"), dict) - else 0 - ), - 0, - ), - }, - "alert_thresholds": _summary_int_dict(summary, "alert_thresholds"), - "alert_flags": _summary_bool_dict(summary, "alert_flags"), - } - - -def _queue_item_columns() -> tuple: - return ( - IngestionQueueItem.id, - IngestionQueueItem.scholar_profile_id, - ScholarProfile.display_name, - ScholarProfile.scholar_id, - IngestionQueueItem.status, - IngestionQueueItem.reason, - IngestionQueueItem.dropped_reason, - IngestionQueueItem.attempt_count, - IngestionQueueItem.resume_cstart, - IngestionQueueItem.next_attempt_dt, - IngestionQueueItem.updated_at, - IngestionQueueItem.last_error, - IngestionQueueItem.last_run_id, - ) - - -def _queue_item_select(*, user_id: int): - return ( - select(*_queue_item_columns()) - .join( - ScholarProfile, - and_( - ScholarProfile.id == IngestionQueueItem.scholar_profile_id, - ScholarProfile.user_id == IngestionQueueItem.user_id, - ), - ) - .where(IngestionQueueItem.user_id == user_id) - ) - - -def _queue_list_item_from_row(row: tuple) -> QueueListItem: - ( - item_id, - scholar_profile_id, - display_name, - scholar_id, - status, - reason, - dropped_reason, - attempt_count, - resume_cstart, - next_attempt_dt, - updated_at, - last_error, - last_run_id, - ) = row - return QueueListItem( - id=int(item_id), - scholar_profile_id=int(scholar_profile_id), - scholar_label=(display_name or scholar_id), - status=str(status), - reason=str(reason), - dropped_reason=dropped_reason, - attempt_count=int(attempt_count or 0), - resume_cstart=int(resume_cstart or 0), - next_attempt_dt=next_attempt_dt, - updated_at=updated_at, - last_error=last_error, - last_run_id=int(last_run_id) if last_run_id is not None else None, - ) - - -async def list_recent_runs_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 20, -) -> list[CrawlRun]: - result = await db_session.execute( - select(CrawlRun) - .where(CrawlRun.user_id == user_id) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(limit) - ) - return list(result.scalars().all()) - - -async def list_runs_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 100, - failed_only: bool = False, -) -> list[CrawlRun]: - stmt = ( - select(CrawlRun) - .where(CrawlRun.user_id == user_id) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(limit) - ) - if failed_only: - stmt = stmt.where( - CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE]) - ) - result = await db_session.execute(stmt) - return list(result.scalars().all()) - - -async def get_run_for_user( - db_session: AsyncSession, - *, - user_id: int, - run_id: int, -) -> CrawlRun | None: - result = await db_session.execute( - select(CrawlRun).where( - CrawlRun.user_id == user_id, - CrawlRun.id == run_id, - ) - ) - return result.scalar_one_or_none() - - -async def get_manual_run_by_idempotency_key( - db_session: AsyncSession, - *, - user_id: int, - idempotency_key: str, -) -> CrawlRun | None: - result = await db_session.execute( - select(CrawlRun) - .where( - CrawlRun.user_id == user_id, - CrawlRun.trigger_type == RunTriggerType.MANUAL, - or_( - CrawlRun.idempotency_key == idempotency_key, - CrawlRun.error_log["meta"]["idempotency_key"].astext == idempotency_key, - ), - ) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(1) - ) - return result.scalar_one_or_none() - - -async def list_queue_items_for_user( - db_session: AsyncSession, - *, - user_id: int, - limit: int = 200, -) -> list[QueueListItem]: - result = await db_session.execute( - _queue_item_select(user_id=user_id) - .order_by( - case((IngestionQueueItem.status == "dropped", 1), else_=0).asc(), - IngestionQueueItem.next_attempt_dt.asc(), - IngestionQueueItem.id.asc(), - ) - .limit(limit) - ) - return [_queue_list_item_from_row(row) for row in result.all()] - - -async def get_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueListItem | None: - result = await db_session.execute( - _queue_item_select(user_id=user_id) - .where(IngestionQueueItem.id == queue_item_id) - .limit(1) - ) - row = result.one_or_none() - if row is None: - return None - return _queue_list_item_from_row(row) - - -async def retry_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueListItem | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.id == queue_item_id, - IngestionQueueItem.user_id == user_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status == QUEUE_STATUS_QUEUED: - raise QueueTransitionError( - code="queue_item_already_queued", - message="Queue item is already queued.", - current_status=item.status, - ) - if item.status == QUEUE_STATUS_RETRYING: - raise QueueTransitionError( - code="queue_item_retrying", - message="Queue item is currently retrying.", - current_status=item.status, - ) - - await queue_service.mark_queued_now( - db_session, - job_id=item.id, - reason="manual_retry", - reset_attempt_count=(item.status == QUEUE_STATUS_DROPPED), - ) - await db_session.commit() - - return await get_queue_item_for_user( - db_session, - user_id=user_id, - queue_item_id=queue_item_id, - ) - - -async def drop_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueListItem | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.id == queue_item_id, - IngestionQueueItem.user_id == user_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status == QUEUE_STATUS_DROPPED: - raise QueueTransitionError( - code="queue_item_already_dropped", - message="Queue item is already dropped.", - current_status=item.status, - ) - - await queue_service.mark_dropped( - db_session, - job_id=int(item.id), - reason="manual_drop", - ) - await db_session.commit() - return await get_queue_item_for_user( - db_session, - user_id=user_id, - queue_item_id=queue_item_id, - ) - - -async def clear_queue_item_for_user( - db_session: AsyncSession, - *, - user_id: int, - queue_item_id: int, -) -> QueueClearResult | None: - result = await db_session.execute( - select(IngestionQueueItem).where( - IngestionQueueItem.id == queue_item_id, - IngestionQueueItem.user_id == user_id, - ) - ) - item = result.scalar_one_or_none() - if item is None: - return None - if item.status != QUEUE_STATUS_DROPPED: - raise QueueTransitionError( - code="queue_item_not_dropped", - message="Queue item can only be cleared after it is dropped.", - current_status=item.status, - ) - - item_id = int(item.id) - previous_status = str(item.status) - deleted = await queue_service.delete_job_by_id( - db_session, - job_id=item_id, - ) - await db_session.commit() - if not deleted: - return None - return QueueClearResult( - queue_item_id=item_id, - previous_status=previous_status, - ) - - -async def queue_status_counts_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> dict[str, int]: - result = await db_session.execute( - select( - IngestionQueueItem.status, - func.count(IngestionQueueItem.id), - ) - .where(IngestionQueueItem.user_id == user_id) - .group_by(IngestionQueueItem.status) - ) - counts: dict[str, int] = {"queued": 0, "retrying": 0, "dropped": 0} - for status, count in result.all(): - counts[str(status)] = int(count or 0) - return counts +from app.services.domains.runs.application import * diff --git a/app/services/scheduler.py b/app/services/scheduler.py index 2057663..15b5656 100644 --- a/app/services/scheduler.py +++ b/app/services/scheduler.py @@ -1,582 +1,3 @@ from __future__ import annotations -import asyncio -from dataclasses import dataclass -from datetime import datetime, timedelta, timezone -import logging - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import ( - CrawlRun, - QueueItemStatus, - RunTriggerType, - ScholarProfile, - User, - UserSetting, -) -from app.db.session import get_session_factory -from app.services import continuation_queue as queue_service -from app.services.ingestion import ( - RunAlreadyInProgressError, - RunBlockedBySafetyPolicyError, - ScholarIngestionService, -) -from app.services.scholar_source import LiveScholarSource -from app.settings import settings - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class _AutoRunCandidate: - user_id: int - run_interval_minutes: int - request_delay_seconds: int - cooldown_until: datetime | None - cooldown_reason: str | None - - -class SchedulerService: - def __init__( - self, - *, - enabled: bool, - tick_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - continuation_queue_enabled: bool, - continuation_base_delay_seconds: int, - continuation_max_delay_seconds: int, - continuation_max_attempts: int, - queue_batch_size: int, - ) -> None: - self._enabled = enabled - self._tick_seconds = max(5, int(tick_seconds)) - self._network_error_retries = max(0, int(network_error_retries)) - self._retry_backoff_seconds = max(0.0, float(retry_backoff_seconds)) - self._max_pages_per_scholar = max(1, int(max_pages_per_scholar)) - self._page_size = max(1, int(page_size)) - self._continuation_queue_enabled = bool(continuation_queue_enabled) - self._continuation_base_delay_seconds = max(1, int(continuation_base_delay_seconds)) - self._continuation_max_delay_seconds = max( - self._continuation_base_delay_seconds, - int(continuation_max_delay_seconds), - ) - self._continuation_max_attempts = max(1, int(continuation_max_attempts)) - self._queue_batch_size = max(1, int(queue_batch_size)) - self._task: asyncio.Task[None] | None = None - self._source = LiveScholarSource() - - async def start(self) -> None: - if not self._enabled: - logger.info( - "scheduler.disabled", - extra={ - "event": "scheduler.disabled", - }, - ) - return - if self._task is not None: - return - self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler") - logger.info( - "scheduler.started", - extra={ - "event": "scheduler.started", - "tick_seconds": self._tick_seconds, - "network_error_retries": self._network_error_retries, - "retry_backoff_seconds": self._retry_backoff_seconds, - "max_pages_per_scholar": self._max_pages_per_scholar, - "page_size": self._page_size, - "continuation_queue_enabled": self._continuation_queue_enabled, - "continuation_base_delay_seconds": self._continuation_base_delay_seconds, - "continuation_max_delay_seconds": self._continuation_max_delay_seconds, - "continuation_max_attempts": self._continuation_max_attempts, - "queue_batch_size": self._queue_batch_size, - }, - ) - - async def stop(self) -> None: - if self._task is None: - return - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - finally: - self._task = None - logger.info("scheduler.stopped", extra={"event": "scheduler.stopped"}) - - async def _run_loop(self) -> None: - while True: - try: - await self._tick_once() - except asyncio.CancelledError: - raise - except Exception: - logger.exception( - "scheduler.tick_failed", - extra={ - "event": "scheduler.tick_failed", - }, - ) - await asyncio.sleep(float(self._tick_seconds)) - - async def _tick_once(self) -> None: - if self._continuation_queue_enabled: - await self._drain_continuation_queue() - candidates = await self._load_candidates() - if not candidates: - return - now = datetime.now(timezone.utc) - for candidate in candidates: - if not await self._is_due(candidate, now=now): - continue - await self._run_candidate(candidate) - - async def _load_candidate_rows(self) -> list[tuple]: - session_factory = get_session_factory() - async with session_factory() as session: - result = await session.execute( - select( - UserSetting.user_id, - UserSetting.run_interval_minutes, - UserSetting.request_delay_seconds, - UserSetting.scrape_cooldown_until, - UserSetting.scrape_cooldown_reason, - ) - .join(User, User.id == UserSetting.user_id) - .where(User.is_active.is_(True), UserSetting.auto_run_enabled.is_(True)) - .order_by(UserSetting.user_id.asc()) - ) - return result.all() - - @staticmethod - def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None: - user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row - if cooldown_until is not None and cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - if cooldown_until is not None and cooldown_until > now_utc: - logger.info( - "scheduler.run_skipped_safety_cooldown_precheck", - extra={ - "event": "scheduler.run_skipped_safety_cooldown_precheck", - "user_id": int(user_id), - "reason": cooldown_reason, - "cooldown_until": cooldown_until, - "cooldown_remaining_seconds": int((cooldown_until - now_utc).total_seconds()), - "metric_name": "scheduler_run_skipped_safety_cooldown_total", - "metric_value": 1, - }, - ) - return None - return _AutoRunCandidate( - user_id=int(user_id), - run_interval_minutes=int(run_interval_minutes), - request_delay_seconds=int(request_delay_seconds), - cooldown_until=cooldown_until, - cooldown_reason=(str(cooldown_reason).strip() if cooldown_reason else None), - ) - - async def _load_candidates(self) -> list[_AutoRunCandidate]: - if not settings.ingestion_automation_allowed: - return [] - rows = await self._load_candidate_rows() - now_utc = datetime.now(timezone.utc) - candidates: list[_AutoRunCandidate] = [] - for row in rows: - candidate = self._candidate_from_row(row, now_utc=now_utc) - if candidate is not None: - candidates.append(candidate) - return candidates - - async def _is_due(self, candidate: _AutoRunCandidate, *, now: datetime) -> bool: - session_factory = get_session_factory() - async with session_factory() as session: - result = await session.execute( - select(CrawlRun.start_dt) - .where( - CrawlRun.user_id == candidate.user_id, - ) - .order_by(CrawlRun.start_dt.desc(), CrawlRun.id.desc()) - .limit(1) - ) - last_run = result.scalar_one_or_none() - - if last_run is None: - return True - - next_due_dt = last_run + timedelta( - minutes=candidate.run_interval_minutes - ) - return now >= next_due_dt - - async def _run_candidate_ingestion( - self, - *, - candidate: _AutoRunCandidate, - ): - session_factory = get_session_factory() - async with session_factory() as session: - ingestion = ScholarIngestionService(source=self._source) - try: - return await ingestion.run_for_user( - session, - user_id=candidate.user_id, - trigger_type=RunTriggerType.SCHEDULED, - request_delay_seconds=candidate.request_delay_seconds, - network_error_retries=self._network_error_retries, - retry_backoff_seconds=self._retry_backoff_seconds, - max_pages_per_scholar=self._max_pages_per_scholar, - page_size=self._page_size, - auto_queue_continuations=self._continuation_queue_enabled, - queue_delay_seconds=self._continuation_base_delay_seconds, - alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, - alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, - alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, - ) - except RunAlreadyInProgressError: - await session.rollback() - logger.info("scheduler.run_skipped_locked", extra={"event": "scheduler.run_skipped_locked", "user_id": candidate.user_id}) - return None - except RunBlockedBySafetyPolicyError as exc: - await session.rollback() - logger.info( - "scheduler.run_skipped_safety_cooldown", - extra={ - "event": "scheduler.run_skipped_safety_cooldown", - "user_id": candidate.user_id, - "reason": exc.safety_state.get("cooldown_reason"), - "cooldown_until": exc.safety_state.get("cooldown_until"), - "cooldown_remaining_seconds": exc.safety_state.get("cooldown_remaining_seconds"), - "metric_name": "scheduler_run_skipped_safety_cooldown_total", - "metric_value": 1, - }, - ) - return None - except Exception: - await session.rollback() - logger.exception("scheduler.run_failed", extra={"event": "scheduler.run_failed", "user_id": candidate.user_id}) - return None - - async def _run_candidate(self, candidate: _AutoRunCandidate) -> None: - run_summary = await self._run_candidate_ingestion(candidate=candidate) - if run_summary is None: - return - logger.info( - "scheduler.run_completed", - extra={ - "event": "scheduler.run_completed", - "user_id": candidate.user_id, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - "scholar_count": run_summary.scholar_count, - "new_publication_count": run_summary.new_publication_count, - }, - ) - - async def _drain_continuation_queue(self) -> None: - now = datetime.now(timezone.utc) - session_factory = get_session_factory() - async with session_factory() as session: - jobs = await queue_service.list_due_jobs( - session, - now=now, - limit=self._queue_batch_size, - ) - for job in jobs: - await self._run_queue_job(job) - - async def _drop_queue_job_if_max_attempts( - self, - job: queue_service.ContinuationQueueJob, - ) -> bool: - if job.attempt_count < self._continuation_max_attempts: - return False - session_factory = get_session_factory() - async with session_factory() as session: - dropped = await queue_service.mark_dropped( - session, - job_id=job.id, - reason="max_attempts_reached", - ) - await session.commit() - if dropped is not None: - logger.warning( - "scheduler.queue_item_dropped_max_attempts", - extra={ - "event": "scheduler.queue_item_dropped_max_attempts", - "queue_item_id": job.id, - "user_id": job.user_id, - "scholar_profile_id": job.scholar_profile_id, - "attempt_count": job.attempt_count, - "max_attempts": self._continuation_max_attempts, - }, - ) - return True - - async def _mark_queue_job_retrying( - self, - job: queue_service.ContinuationQueueJob, - ) -> bool: - session_factory = get_session_factory() - async with session_factory() as session: - queue_item = await queue_service.mark_retrying(session, job_id=job.id) - await session.commit() - if queue_item is None: - return False - if queue_item.status == QueueItemStatus.DROPPED.value: - return False - return True - - async def _queue_job_has_available_scholar( - self, - job: queue_service.ContinuationQueueJob, - ) -> bool: - session_factory = get_session_factory() - async with session_factory() as session: - scholar_result = await session.execute( - select(ScholarProfile.id).where( - ScholarProfile.user_id == job.user_id, - ScholarProfile.id == job.scholar_profile_id, - ScholarProfile.is_enabled.is_(True), - ) - ) - scholar_id = scholar_result.scalar_one_or_none() - if scholar_id is not None: - return True - dropped = await queue_service.mark_dropped( - session, - job_id=job.id, - reason="scholar_unavailable", - ) - await session.commit() - if dropped is not None: - logger.info( - "scheduler.queue_item_dropped_scholar_unavailable", - extra={ - "event": "scheduler.queue_item_dropped_scholar_unavailable", - "queue_item_id": job.id, - "user_id": job.user_id, - "scholar_profile_id": job.scholar_profile_id, - }, - ) - return False - - async def _reschedule_queue_job_lock_active(self, job: queue_service.ContinuationQueueJob) -> None: - session_factory = get_session_factory() - async with session_factory() as recovery_session: - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=max(self._tick_seconds, 15), - reason="user_run_lock_active", - error="run_already_in_progress", - ) - await recovery_session.commit() - logger.info( - "scheduler.queue_item_deferred_lock", - extra={"event": "scheduler.queue_item_deferred_lock", "queue_item_id": job.id, "user_id": job.user_id}, - ) - - async def _reschedule_queue_job_safety_cooldown( - self, - job: queue_service.ContinuationQueueJob, - exc: RunBlockedBySafetyPolicyError, - ) -> None: - cooldown_remaining_seconds = max( - self._tick_seconds, - int(exc.safety_state.get("cooldown_remaining_seconds") or 0), - ) - session_factory = get_session_factory() - async with session_factory() as recovery_session: - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=max(self._tick_seconds, cooldown_remaining_seconds), - reason="scrape_safety_cooldown", - error=str(exc.message), - ) - await recovery_session.commit() - logger.info( - "scheduler.queue_item_deferred_safety_cooldown", - extra={ - "event": "scheduler.queue_item_deferred_safety_cooldown", - "queue_item_id": job.id, - "user_id": job.user_id, - "reason": exc.safety_state.get("cooldown_reason"), - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "metric_name": "scheduler_queue_item_deferred_safety_cooldown_total", - "metric_value": 1, - }, - ) - - async def _reschedule_queue_job_after_exception( - self, - job: queue_service.ContinuationQueueJob, - *, - exc: Exception, - ) -> None: - session_factory = get_session_factory() - async with session_factory() as recovery_session: - queue_item = await queue_service.increment_attempt_count(recovery_session, job_id=job.id) - if queue_item is None: - await recovery_session.commit() - return - if int(queue_item.attempt_count) >= self._continuation_max_attempts: - await queue_service.mark_dropped( - recovery_session, - job_id=job.id, - reason="scheduler_exception_max_attempts", - error=str(exc), - ) - await recovery_session.commit() - logger.warning( - "scheduler.queue_item_dropped_after_exception", - extra={"event": "scheduler.queue_item_dropped_after_exception", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count}, - ) - return - delay_seconds = queue_service.compute_backoff_seconds( - base_seconds=self._continuation_base_delay_seconds, - attempt_count=int(queue_item.attempt_count), - max_seconds=self._continuation_max_delay_seconds, - ) - await queue_service.reschedule_job( - recovery_session, - job_id=job.id, - delay_seconds=delay_seconds, - reason="scheduler_exception", - error=str(exc), - ) - await recovery_session.commit() - logger.exception("scheduler.queue_item_run_failed", extra={"event": "scheduler.queue_item_run_failed", "queue_item_id": job.id, "user_id": job.user_id}) - - async def _run_ingestion_for_queue_job( - self, - job: queue_service.ContinuationQueueJob, - ): - session_factory = get_session_factory() - async with session_factory() as session: - request_delay_seconds = await self._load_request_delay_for_user(session, user_id=job.user_id) - ingestion = ScholarIngestionService(source=self._source) - try: - return await ingestion.run_for_user( - session, - user_id=job.user_id, - trigger_type=RunTriggerType.SCHEDULED, - request_delay_seconds=request_delay_seconds, - network_error_retries=self._network_error_retries, - retry_backoff_seconds=self._retry_backoff_seconds, - max_pages_per_scholar=self._max_pages_per_scholar, - page_size=self._page_size, - scholar_profile_ids={job.scholar_profile_id}, - start_cstart_by_scholar_id={job.scholar_profile_id: job.resume_cstart}, - auto_queue_continuations=self._continuation_queue_enabled, - queue_delay_seconds=self._continuation_base_delay_seconds, - alert_blocked_failure_threshold=settings.ingestion_alert_blocked_failure_threshold, - alert_network_failure_threshold=settings.ingestion_alert_network_failure_threshold, - alert_retry_scheduled_threshold=settings.ingestion_alert_retry_scheduled_threshold, - ) - except RunAlreadyInProgressError: - await session.rollback() - await self._reschedule_queue_job_lock_active(job) - except RunBlockedBySafetyPolicyError as exc: - await session.rollback() - await self._reschedule_queue_job_safety_cooldown(job, exc) - except Exception as exc: - await session.rollback() - await self._reschedule_queue_job_after_exception(job, exc=exc) - return None - - @staticmethod - def _log_queue_item_resolved( - *, - event_name: str, - job: queue_service.ContinuationQueueJob, - run_summary, - attempt_count: int | None = None, - delay_seconds: int | None = None, - ) -> None: - payload = { - "event": event_name, - "queue_item_id": job.id, - "user_id": job.user_id, - "run_id": run_summary.crawl_run_id, - "status": run_summary.status.value, - } - if attempt_count is not None: - payload["attempt_count"] = attempt_count - if delay_seconds is not None: - payload["delay_seconds"] = delay_seconds - logger.info(event_name, extra=payload) - - async def _finalize_queue_job_after_run(self, job: queue_service.ContinuationQueueJob, run_summary) -> None: - session_factory = get_session_factory() - async with session_factory() as session: - if int(run_summary.failed_count) <= 0: - queue_item = await queue_service.reset_attempt_count(session, job_id=job.id) - await session.commit() - if queue_item is None: - self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) - return - self._log_queue_item_resolved( - event_name="scheduler.queue_item_progressed", - job=job, - run_summary=run_summary, - attempt_count=int(queue_item.attempt_count), - ) - return - queue_item = await queue_service.increment_attempt_count(session, job_id=job.id) - if queue_item is None: - await session.commit() - self._log_queue_item_resolved(event_name="scheduler.queue_item_resolved", job=job, run_summary=run_summary) - return - if int(queue_item.attempt_count) >= self._continuation_max_attempts: - await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run") - await session.commit() - logger.warning( - "scheduler.queue_item_dropped_max_attempts_after_run", - extra={"event": "scheduler.queue_item_dropped_max_attempts_after_run", "queue_item_id": job.id, "user_id": job.user_id, "attempt_count": queue_item.attempt_count, "run_id": run_summary.crawl_run_id, "status": run_summary.status.value}, - ) - return - delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds) - await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error) - await session.commit() - self._log_queue_item_resolved( - event_name="scheduler.queue_item_rescheduled_failed", - job=job, - run_summary=run_summary, - attempt_count=int(queue_item.attempt_count), - delay_seconds=delay_seconds, - ) - - async def _run_queue_job(self, job: queue_service.ContinuationQueueJob) -> None: - if await self._drop_queue_job_if_max_attempts(job): - return - if not await self._mark_queue_job_retrying(job): - return - if not await self._queue_job_has_available_scholar(job): - return - run_summary = await self._run_ingestion_for_queue_job(job) - if run_summary is None: - return - await self._finalize_queue_job_after_run(job, run_summary) - - async def _load_request_delay_for_user( - self, - db_session: AsyncSession, - *, - user_id: int, - ) -> int: - result = await db_session.execute( - select(UserSetting.request_delay_seconds).where(UserSetting.user_id == user_id) - ) - delay = result.scalar_one_or_none() - if delay is None: - return 10 - return max(1, int(delay)) +from app.services.domains.ingestion.scheduler import * diff --git a/app/services/scholar_parser.py b/app/services/scholar_parser.py index 8f357d0..c31db46 100644 --- a/app/services/scholar_parser.py +++ b/app/services/scholar_parser.py @@ -1,886 +1,3 @@ from __future__ import annotations -import re -from dataclasses import dataclass -from enum import StrEnum -from html import unescape -from html.parser import HTMLParser -from typing import Any -from urllib.parse import parse_qs, urljoin, urlparse - -from app.services.scholar_source import FetchResult - -BLOCKED_KEYWORDS = [ - "unusual traffic", - "sorry/index", - "not a robot", - "our systems have detected", - "automated queries", - "recaptcha", - "captcha", -] - -NO_RESULTS_KEYWORDS = [ - "didn't match any articles", - "did not match any articles", - "no articles", - "no documents", -] - -NO_AUTHOR_RESULTS_KEYWORDS = [ - "didn't match any user profiles", - "did not match any user profiles", - "didn't match any scholars", - "did not match any scholars", - "no user profiles", -] - -MARKER_KEYS = [ - "gsc_a_tr", - "gsc_a_at", - "gsc_a_ac", - "gsc_a_h", - "gsc_a_y", - "gs_gray", - "gsc_prf_in", - "gsc_rsb_st", -] - -AUTHOR_SEARCH_MARKER_KEYS = [ - "gsc_1usr", - "gs_ai_name", - "gs_ai_aff", - "gs_ai_eml", - "gs_ai_cby", - "gs_ai_one_int", -] - -NETWORK_DNS_ERROR_KEYWORDS = [ - "temporary failure in name resolution", - "name or service not known", - "nodename nor servname provided", - "getaddrinfo failed", -] - -NETWORK_TIMEOUT_KEYWORDS = [ - "timed out", - "timeout", -] - -NETWORK_TLS_ERROR_KEYWORDS = [ - "ssl", - "tls", - "certificate verify failed", -] - -TAG_RE = re.compile(r"<[^>]+>", re.S) -SCRIPT_STYLE_RE = re.compile(r"<(script|style)\b[^>]*>.*?", re.I | re.S) -SHOW_MORE_BUTTON_RE = re.compile( - r"]*\bid\s*=\s*['\"]gsc_bpf_more['\"][^>]*>", - re.I | re.S, -) -PROFILE_ROW_PARSER_DIRECT_MARKERS = ( - "gs_ggs", - "gs_ggsd", - "gs_ggsa", - "gs_or_ggsm", -) -PROFILE_ROW_DIRECT_LABEL_TOKENS = ( - "pdf", - "[pdf]", - "full text", - "download", -) - - -class ParseState(StrEnum): - OK = "ok" - NO_RESULTS = "no_results" - BLOCKED_OR_CAPTCHA = "blocked_or_captcha" - LAYOUT_CHANGED = "layout_changed" - NETWORK_ERROR = "network_error" - - -@dataclass(frozen=True) -class PublicationCandidate: - title: str - title_url: str | None - cluster_id: str | None - year: int | None - citation_count: int | None - authors_text: str | None - venue_text: str | None - pdf_url: str | None - - -@dataclass(frozen=True) -class ScholarSearchCandidate: - scholar_id: str - display_name: str - affiliation: str | None - email_domain: str | None - cited_by_count: int | None - interests: list[str] - profile_url: str - profile_image_url: str | None - - -@dataclass(frozen=True) -class ParsedProfilePage: - state: ParseState - state_reason: str - profile_name: str | None - profile_image_url: str | None - publications: list[PublicationCandidate] - marker_counts: dict[str, int] - warnings: list[str] - has_show_more_button: bool - has_operation_error_banner: bool - articles_range: str | None - - -@dataclass(frozen=True) -class ParsedAuthorSearchPage: - state: ParseState - state_reason: str - candidates: list[ScholarSearchCandidate] - marker_counts: dict[str, int] - warnings: list[str] - - -def normalize_space(value: str) -> str: - return " ".join(unescape(value).split()) - - -def strip_tags(value: str) -> str: - return normalize_space(TAG_RE.sub(" ", value)) - - -def attr_class(attrs: list[tuple[str, str | None]]) -> str: - for name, raw_value in attrs: - if name.lower() == "class": - return raw_value or "" - return "" - - -def attr_href(attrs: list[tuple[str, str | None]]) -> str | None: - for name, raw_value in attrs: - if name.lower() == "href": - return raw_value - return None - - -def attr_src(attrs: list[tuple[str, str | None]]) -> str | None: - for name, raw_value in attrs: - if name.lower() == "src": - return raw_value - return None - - -def build_absolute_scholar_url(path_or_url: str | None) -> str | None: - if not path_or_url: - return None - return urljoin("https://scholar.google.com", path_or_url) - - -class ScholarRowParser(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.title_href: str | None = None - self.direct_download_href: str | None = None - self.title_parts: list[str] = [] - self.citation_parts: list[str] = [] - self.year_parts: list[str] = [] - self.gray_texts: list[str] = [] - - self._title_depth = 0 - self._citation_depth = 0 - self._year_depth = 0 - self._gray_stack: list[dict[str, Any]] = [] - self._direct_marker_depth = 0 - self._aux_link_stack: list[dict[str, Any]] = [] - - @staticmethod - def _contains_direct_marker(classes: str) -> bool: - lowered = classes.lower() - return any(marker in lowered for marker in PROFILE_ROW_PARSER_DIRECT_MARKERS) - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if self._title_depth > 0: - self._title_depth += 1 - if self._citation_depth > 0: - self._citation_depth += 1 - if self._year_depth > 0: - self._year_depth += 1 - if self._gray_stack: - self._gray_stack[-1]["depth"] += 1 - if self._direct_marker_depth > 0: - self._direct_marker_depth += 1 - if self._aux_link_stack: - self._aux_link_stack[-1]["depth"] += 1 - - classes = attr_class(attrs) - if tag in {"div", "span"} and self._contains_direct_marker(classes): - self._direct_marker_depth = 1 - - if tag == "a" and "gsc_a_at" in classes: - self._title_depth = 1 - self.title_href = attr_href(attrs) - return - - if tag == "a" and "gsc_a_ac" in classes: - self._citation_depth = 1 - return - - if tag in {"span", "a"} and ("gsc_a_h" in classes or "gsc_a_y" in classes): - self._year_depth = 1 - return - - if tag == "div" and "gs_gray" in classes: - self._gray_stack.append({"depth": 1, "parts": []}) - return - - if tag == "a": - self._aux_link_stack.append( - { - "depth": 1, - "href": attr_href(attrs), - "classes": classes, - "parts": [], - } - ) - - def handle_data(self, data: str) -> None: - if self._title_depth > 0: - self.title_parts.append(data) - if self._citation_depth > 0: - self.citation_parts.append(data) - if self._year_depth > 0: - self.year_parts.append(data) - if self._gray_stack: - self._gray_stack[-1]["parts"].append(data) - if self._aux_link_stack: - self._aux_link_stack[-1]["parts"].append(data) - - def _capture_direct_download_href(self, link: dict[str, Any]) -> None: - if self.direct_download_href: - return - href = link.get("href") - if not isinstance(href, str) or not href.strip(): - return - label = normalize_space("".join(link.get("parts", []))).lower() - classes = str(link.get("classes", "")).lower() - label_match = any(token in label for token in PROFILE_ROW_DIRECT_LABEL_TOKENS) - marker_match = self._contains_direct_marker(classes) or self._direct_marker_depth > 0 - if label_match or marker_match: - self.direct_download_href = href.strip() - - def handle_endtag(self, _tag: str) -> None: - if self._title_depth > 0: - self._title_depth -= 1 - if self._citation_depth > 0: - self._citation_depth -= 1 - if self._year_depth > 0: - self._year_depth -= 1 - if self._gray_stack: - self._gray_stack[-1]["depth"] -= 1 - if self._gray_stack[-1]["depth"] == 0: - text = normalize_space("".join(self._gray_stack[-1]["parts"])) - if text: - self.gray_texts.append(text) - self._gray_stack.pop() - if self._aux_link_stack: - self._aux_link_stack[-1]["depth"] -= 1 - if self._aux_link_stack[-1]["depth"] == 0: - self._capture_direct_download_href(self._aux_link_stack[-1]) - self._aux_link_stack.pop() - if self._direct_marker_depth > 0: - self._direct_marker_depth -= 1 - - -def extract_rows(html: str) -> list[str]: - pattern = re.compile( - r"]*\bclass\s*=\s*['\"][^'\"]*\bgsc_a_tr\b[^'\"]*['\"])[^>]*>(.*?)", - re.I | re.S, - ) - return [match.group(1) for match in pattern.finditer(html)] - - -def parse_cluster_id_from_href(href: str | None) -> str | None: - if not href: - return None - parsed = urlparse(href) - query = parse_qs(parsed.query) - - citation_for_view = query.get("citation_for_view") - if citation_for_view: - token = citation_for_view[0].strip() - if token: - if ":" in token: - return token.rsplit(":", 1)[-1] or None - return token - - cluster = query.get("cluster") - if cluster: - token = cluster[0].strip() - if token: - return token - return None - - -def parse_scholar_id_from_href(href: str | None) -> str | None: - if not href: - return None - parsed = urlparse(href) - query = parse_qs(parsed.query) - user_values = query.get("user") - if not user_values: - return None - candidate = user_values[0].strip() - return candidate or None - - -def parse_year(parts: list[str]) -> int | None: - text = normalize_space(" ".join(parts)) - match = re.search(r"\b(19|20)\d{2}\b", text) - if not match: - return None - try: - return int(match.group(0)) - except ValueError: - return None - - -def parse_citation_count(parts: list[str]) -> int | None: - text = normalize_space(" ".join(parts)) - if not text: - return 0 - match = re.search(r"\d+", text) - if not match: - return None - return int(match.group(0)) - - -def _parse_publication_row(row_html: str) -> tuple[PublicationCandidate | None, list[str]]: - parser = ScholarRowParser() - parser.feed(row_html) - warnings: list[str] = [] - title = normalize_space("".join(parser.title_parts)) - if not title: - warnings.append("row_missing_title") - return None, warnings - if not parser.title_href: - warnings.append("row_missing_title_href") - - citation_text = normalize_space(" ".join(parser.citation_parts)) - citation_count = parse_citation_count(parser.citation_parts) - if citation_text and citation_count is None: - warnings.append("layout_row_citation_unparseable") - - year_text = normalize_space(" ".join(parser.year_parts)) - year = parse_year(parser.year_parts) - if year_text and year is None: - warnings.append("layout_row_year_unparseable") - - authors_text = parser.gray_texts[0] if len(parser.gray_texts) > 0 else None - venue_text = parser.gray_texts[1] if len(parser.gray_texts) > 1 else None - return ( - PublicationCandidate( - title=title, - title_url=parser.title_href, - cluster_id=parse_cluster_id_from_href(parser.title_href), - year=year, - citation_count=citation_count, - authors_text=authors_text, - venue_text=venue_text, - pdf_url=build_absolute_scholar_url(parser.direct_download_href), - ), - warnings, - ) - - -def parse_publications(html: str) -> tuple[list[PublicationCandidate], list[str]]: - rows = extract_rows(html) - warnings: list[str] = [] - publications: list[PublicationCandidate] = [] - - for row_html in rows: - publication, row_warnings = _parse_publication_row(row_html) - warnings.extend(row_warnings) - if publication is None: - continue - publications.append(publication) - - if not rows: - warnings.append("no_rows_detected") - if rows and not publications: - warnings.append("layout_all_rows_unparseable") - - return publications, sorted(set(warnings)) - - -def extract_profile_name(html: str) -> str | None: - pattern = re.compile( - r"<[^>]*\bid\s*=\s*['\"]gsc_prf_in['\"][^>]*>(.*?)]+>", - re.I | re.S, - ) - match = pattern.search(html) - if not match: - return None - value = strip_tags(match.group(1)) - return value or None - - -def extract_profile_image_url(html: str) -> str | None: - og_image_pattern = re.compile( - r"]+property=['\"]og:image['\"][^>]+content=['\"]([^'\"]+)['\"][^>]*>", - re.I | re.S, - ) - og_match = og_image_pattern.search(html) - if og_match: - value = normalize_space(og_match.group(1)) - absolute = build_absolute_scholar_url(value) - if absolute: - return absolute - - image_pattern = re.compile( - r"]*\bid=['\"]gsc_prf_pup-img['\"][^>]*\bsrc=['\"]([^'\"]+)['\"][^>]*>", - re.I | re.S, - ) - image_match = image_pattern.search(html) - if not image_match: - return None - - value = normalize_space(image_match.group(1)) - return build_absolute_scholar_url(value) - - -def extract_articles_range(html: str) -> str | None: - pattern = re.compile( - r"<[^>]*\bid\s*=\s*['\"]gsc_a_nn['\"][^>]*>(.*?)]+>", - re.I | re.S, - ) - match = pattern.search(html) - if not match: - return None - value = strip_tags(match.group(1)) - return value or None - - -def has_show_more_button(html: str) -> bool: - match = SHOW_MORE_BUTTON_RE.search(html) - if match is None: - return False - - button_tag = match.group(0).lower() - if "disabled" in button_tag: - return False - if 'aria-disabled="true"' in button_tag or "aria-disabled='true'" in button_tag: - return False - if "gs_dis" in button_tag: - return False - return True - - -def has_operation_error_banner(html: str) -> bool: - lowered = html.lower() - if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered: - return False - return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered - - -def count_markers(html: str) -> dict[str, int]: - lowered = html.lower() - return {key: lowered.count(key.lower()) for key in MARKER_KEYS} - - -def count_author_search_markers(html: str) -> dict[str, int]: - lowered = html.lower() - return {key: lowered.count(key.lower()) for key in AUTHOR_SEARCH_MARKER_KEYS} - - -def _extract_verified_email_domain(value: str | None) -> str | None: - if not value: - return None - match = re.search(r"verified email at\s+(.+)$", value.strip(), re.I) - if not match: - return None - domain = normalize_space(match.group(1)) - return domain or None - - -class ScholarAuthorSearchParser(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.candidates: list[ScholarSearchCandidate] = [] - self._candidate: dict[str, Any] | None = None - - def _begin_candidate(self) -> None: - self._candidate = { - "depth": 1, - "name_href": None, - "name_parts": [], - "aff_depth": 0, - "aff_parts": [], - "name_depth": 0, - "eml_depth": 0, - "eml_parts": [], - "cby_depth": 0, - "cby_parts": [], - "interest_depth": 0, - "interest_parts": [], - "interests": [], - "image_src": None, - } - - def _increment_capture_depths(self) -> None: - if self._candidate is None: - return - for key in ("name_depth", "aff_depth", "eml_depth", "cby_depth", "interest_depth"): - if self._candidate[key] > 0: - self._candidate[key] += 1 - - def _finalize_candidate(self) -> None: - if self._candidate is None: - return - - name = normalize_space("".join(self._candidate["name_parts"])) - scholar_id = parse_scholar_id_from_href(self._candidate["name_href"]) - if not name or not scholar_id: - return - - affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None - email_domain = _extract_verified_email_domain( - normalize_space("".join(self._candidate["eml_parts"])) or None - ) - cited_by_text = normalize_space("".join(self._candidate["cby_parts"])) - cited_by_match = re.search(r"\d+", cited_by_text) - cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None - - seen_interests: set[str] = set() - interests: list[str] = [] - for interest in self._candidate["interests"]: - normalized = normalize_space(interest) - if not normalized or normalized in seen_interests: - continue - seen_interests.add(normalized) - interests.append(normalized) - - profile_url = build_absolute_scholar_url(self._candidate["name_href"]) - if not profile_url: - profile_url = ( - "https://scholar.google.com/citations" - f"?hl=en&user={scholar_id}" - ) - - self.candidates.append( - ScholarSearchCandidate( - scholar_id=scholar_id, - display_name=name, - affiliation=affiliation, - email_domain=email_domain, - cited_by_count=cited_by_count, - interests=interests, - profile_url=profile_url, - profile_image_url=build_absolute_scholar_url(self._candidate["image_src"]), - ) - ) - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - classes = attr_class(attrs) - - if self._candidate is None: - if tag == "div" and "gsc_1usr" in classes: - self._begin_candidate() - return - - self._candidate["depth"] += 1 - self._increment_capture_depths() - - if tag == "a" and "gs_ai_name" in classes: - self._candidate["name_depth"] = 1 - self._candidate["name_href"] = attr_href(attrs) - return - - if tag == "div" and "gs_ai_aff" in classes: - self._candidate["aff_depth"] = 1 - return - - if tag == "div" and "gs_ai_eml" in classes: - self._candidate["eml_depth"] = 1 - return - - if tag == "div" and "gs_ai_cby" in classes: - self._candidate["cby_depth"] = 1 - return - - if tag == "a" and "gs_ai_one_int" in classes: - self._candidate["interest_depth"] = 1 - self._candidate["interest_parts"] = [] - return - - if tag == "img" and self._candidate["image_src"] is None: - self._candidate["image_src"] = attr_src(attrs) - - def handle_data(self, data: str) -> None: - if self._candidate is None: - return - if self._candidate["name_depth"] > 0: - self._candidate["name_parts"].append(data) - if self._candidate["aff_depth"] > 0: - self._candidate["aff_parts"].append(data) - if self._candidate["eml_depth"] > 0: - self._candidate["eml_parts"].append(data) - if self._candidate["cby_depth"] > 0: - self._candidate["cby_parts"].append(data) - if self._candidate["interest_depth"] > 0: - self._candidate["interest_parts"].append(data) - - def _decrement_capture_depth(self, key: str) -> bool: - if self._candidate is None: - return False - if self._candidate[key] <= 0: - return False - self._candidate[key] -= 1 - return self._candidate[key] == 0 - - def handle_endtag(self, _tag: str) -> None: - if self._candidate is None: - return - - interest_closed = self._decrement_capture_depth("interest_depth") - self._decrement_capture_depth("name_depth") - self._decrement_capture_depth("aff_depth") - self._decrement_capture_depth("eml_depth") - self._decrement_capture_depth("cby_depth") - - if interest_closed: - interest_text = normalize_space("".join(self._candidate["interest_parts"])) - if interest_text: - self._candidate["interests"].append(interest_text) - self._candidate["interest_parts"] = [] - - self._candidate["depth"] -= 1 - if self._candidate["depth"] > 0: - return - - self._finalize_candidate() - self._candidate = None - - -def classify_network_error_reason(fetch_error: str | None) -> str: - lowered = (fetch_error or "").lower() - if lowered: - if any(keyword in lowered for keyword in NETWORK_DNS_ERROR_KEYWORDS): - return "network_dns_resolution_failed" - if any(keyword in lowered for keyword in NETWORK_TIMEOUT_KEYWORDS): - return "network_timeout" - if any(keyword in lowered for keyword in NETWORK_TLS_ERROR_KEYWORDS): - return "network_tls_error" - if "connection reset" in lowered: - return "network_connection_reset" - if "connection refused" in lowered: - return "network_connection_refused" - if "network is unreachable" in lowered: - return "network_unreachable" - return "network_error_missing_status_code" - - -def classify_block_or_captcha_reason( - *, - status_code: int, - final_url: str, - body_lowered: str, -) -> str | None: - if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url): - return "blocked_accounts_redirect" - if status_code == 429: - return "blocked_http_429_rate_limited" - if status_code == 403: - if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url: - return "blocked_http_403_captcha_challenge" - return "blocked_http_403_forbidden" - if "sorry/index" in final_url or "sorry/index" in body_lowered: - return "blocked_google_sorry_challenge" - if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered: - return "blocked_unusual_traffic_detected" - if "automated queries" in body_lowered: - return "blocked_automated_queries_detected" - if "not a robot" in body_lowered: - return "blocked_not_a_robot_challenge" - if "recaptcha" in body_lowered: - return "blocked_recaptcha_challenge" - if "captcha" in body_lowered: - return "blocked_captcha_challenge" - if any(keyword in body_lowered for keyword in BLOCKED_KEYWORDS): - return "blocked_keyword_detected" - return None - - -def _warnings_contain(warnings: list[str], code: str) -> bool: - return any(item == code for item in warnings) - - -def _has_layout_row_failure(marker_counts: dict[str, int], warnings: list[str]) -> bool: - if _warnings_contain(warnings, "layout_all_rows_unparseable"): - return True - if marker_counts.get("gsc_a_tr", 0) <= 0: - return False - if _warnings_contain(warnings, "row_missing_title"): - return True - return marker_counts.get("gsc_a_at", 0) <= 0 - - -def _first_layout_warning(warnings: list[str]) -> str | None: - for warning in warnings: - if warning.startswith("layout_"): - return warning - return None - - -def detect_state( - fetch_result: FetchResult, - publications: list[PublicationCandidate], - marker_counts: dict[str, int], - *, - warnings: list[str], - has_show_more_button_flag: bool, - articles_range: str | None, - visible_text: str, -) -> tuple[ParseState, str]: - if fetch_result.status_code is None: - return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) - - lowered = fetch_result.body.lower() - final = (fetch_result.final_url or "").lower() - status_code = int(fetch_result.status_code) - - block_reason = classify_block_or_captcha_reason( - status_code=status_code, - final_url=final, - body_lowered=lowered, - ) - if block_reason is not None: - return ParseState.BLOCKED_OR_CAPTCHA, block_reason - - if not publications and any(keyword in visible_text for keyword in NO_RESULTS_KEYWORDS): - return ParseState.NO_RESULTS, "no_results_keyword_detected" - - layout_warning = _first_layout_warning(warnings) - if layout_warning is not None: - return ParseState.LAYOUT_CHANGED, layout_warning - - if _has_layout_row_failure(marker_counts, warnings): - return ParseState.LAYOUT_CHANGED, "layout_publication_rows_unparseable" - - if has_show_more_button_flag and not articles_range: - return ParseState.LAYOUT_CHANGED, "layout_show_more_without_articles_range" - - if not publications: - has_profile_markers = marker_counts.get("gsc_prf_in", 0) > 0 - has_table_markers = marker_counts.get("gsc_a_tr", 0) > 0 or marker_counts.get("gsc_a_at", 0) > 0 - if not has_profile_markers and not has_table_markers: - return ParseState.LAYOUT_CHANGED, "layout_markers_missing" - return ParseState.OK, "no_rows_with_known_markers" - - return ParseState.OK, "publications_extracted" - - -def detect_author_search_state( - fetch_result: FetchResult, - candidates: list[ScholarSearchCandidate], - marker_counts: dict[str, int], - *, - visible_text: str, -) -> tuple[ParseState, str]: - if fetch_result.status_code is None: - return ParseState.NETWORK_ERROR, classify_network_error_reason(fetch_result.error) - - lowered = fetch_result.body.lower() - final = (fetch_result.final_url or "").lower() - status_code = int(fetch_result.status_code) - - block_reason = classify_block_or_captcha_reason( - status_code=status_code, - final_url=final, - body_lowered=lowered, - ) - if block_reason is not None: - return ParseState.BLOCKED_OR_CAPTCHA, block_reason - - if not candidates and any(keyword in visible_text for keyword in NO_AUTHOR_RESULTS_KEYWORDS): - return ParseState.NO_RESULTS, "no_results_keyword_detected" - - if not candidates: - has_search_markers = marker_counts.get("gsc_1usr", 0) > 0 or marker_counts.get("gs_ai_name", 0) > 0 - if not has_search_markers: - return ParseState.NO_RESULTS, "no_search_candidates_detected" - return ParseState.LAYOUT_CHANGED, "layout_author_candidates_unparseable" - - return ParseState.OK, "author_candidates_extracted" - - -def parse_profile_page(fetch_result: FetchResult) -> ParsedProfilePage: - publications, warnings = parse_publications(fetch_result.body) - marker_counts = count_markers(fetch_result.body) - visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() - - show_more = has_show_more_button(fetch_result.body) - operation_error_banner = has_operation_error_banner(fetch_result.body) - articles_range = extract_articles_range(fetch_result.body) - - if show_more: - warnings.append("possible_partial_page_show_more_present") - if operation_error_banner: - warnings.append("operation_error_banner_present") - - warnings = sorted(set(warnings)) - - state, state_reason = detect_state( - fetch_result, - publications, - marker_counts, - warnings=warnings, - has_show_more_button_flag=show_more, - articles_range=articles_range, - visible_text=visible_text, - ) - - return ParsedProfilePage( - state=state, - state_reason=state_reason, - profile_name=extract_profile_name(fetch_result.body), - profile_image_url=extract_profile_image_url(fetch_result.body), - publications=publications, - marker_counts=marker_counts, - warnings=warnings, - has_show_more_button=show_more, - has_operation_error_banner=operation_error_banner, - articles_range=articles_range, - ) - - -def parse_author_search_page(fetch_result: FetchResult) -> ParsedAuthorSearchPage: - parser = ScholarAuthorSearchParser() - parser.feed(fetch_result.body) - - marker_counts = count_author_search_markers(fetch_result.body) - visible_text = strip_tags(SCRIPT_STYLE_RE.sub(" ", fetch_result.body)).lower() - warnings: list[str] = [] - if not parser.candidates: - warnings.append("no_author_candidates_detected") - - state, state_reason = detect_author_search_state( - fetch_result, - parser.candidates, - marker_counts, - visible_text=visible_text, - ) - - return ParsedAuthorSearchPage( - state=state, - state_reason=state_reason, - candidates=parser.candidates, - marker_counts=marker_counts, - warnings=warnings, - ) +from app.services.domains.scholar.parser import * diff --git a/app/services/scholar_source.py b/app/services/scholar_source.py index 83da165..b4a9147 100644 --- a/app/services/scholar_source.py +++ b/app/services/scholar_source.py @@ -1,225 +1,4 @@ from __future__ import annotations -import asyncio -import logging -import random -from dataclasses import dataclass -from typing import Protocol -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen - -SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations" -DEFAULT_PAGE_SIZE = 100 - -DEFAULT_USER_AGENTS = [ - ( - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" - ), - ( - "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 " - "(KHTML, like Gecko) Version/18.1 Safari/605.1.15" - ), - ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) " - "Gecko/20100101 Firefox/131.0" - ), -] - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class FetchResult: - requested_url: str - status_code: int | None - final_url: str | None - body: str - error: str | None - - -class ScholarSource(Protocol): - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - ... - - async def fetch_profile_page_html( - self, - scholar_id: str, - *, - cstart: int, - pagesize: int, - ) -> FetchResult: - ... - - async def fetch_author_search_html( - self, - query: str, - *, - start: int, - ) -> FetchResult: - ... - - -class LiveScholarSource: - def __init__( - self, - *, - timeout_seconds: float = 25.0, - user_agents: list[str] | None = None, - ) -> None: - self._timeout_seconds = timeout_seconds - self._user_agents = user_agents or DEFAULT_USER_AGENTS - - async def fetch_profile_html(self, scholar_id: str) -> FetchResult: - return await self.fetch_profile_page_html( - scholar_id, - cstart=0, - pagesize=DEFAULT_PAGE_SIZE, - ) - - async def fetch_profile_page_html( - self, - scholar_id: str, - *, - cstart: int, - pagesize: int = DEFAULT_PAGE_SIZE, - ) -> FetchResult: - requested_url = _build_profile_url( - scholar_id=scholar_id, - cstart=cstart, - pagesize=pagesize, - ) - logger.debug( - "scholar_source.fetch_started", - extra={ - "event": "scholar_source.fetch_started", - "scholar_id": scholar_id, - "requested_url": requested_url, - "cstart": cstart, - "pagesize": pagesize, - }, - ) - return await asyncio.to_thread(self._fetch_sync, requested_url) - - async def fetch_author_search_html( - self, - query: str, - *, - start: int = 0, - ) -> FetchResult: - requested_url = _build_author_search_url( - query=query, - start=start, - ) - logger.debug( - "scholar_source.search_fetch_started", - extra={ - "event": "scholar_source.search_fetch_started", - "query": query, - "requested_url": requested_url, - "start": start, - }, - ) - return await asyncio.to_thread(self._fetch_sync, requested_url) - - def _build_request(self, requested_url: str) -> Request: - return Request( - requested_url, - headers={ - "User-Agent": random.choice(self._user_agents), - "Accept": "text/html,application/xhtml+xml", - "Accept-Language": "en-US,en;q=0.9", - "Connection": "close", - }, - ) - - @staticmethod - def _http_error_body(exc: HTTPError) -> str: - try: - return exc.read().decode("utf-8", errors="replace") - except Exception: - return "" - - @staticmethod - def _network_error_result(requested_url: str, exc: URLError) -> FetchResult: - logger.warning( - "scholar_source.fetch_network_error", - extra={"event": "scholar_source.fetch_network_error", "requested_url": requested_url}, - ) - return FetchResult( - requested_url=requested_url, - status_code=None, - final_url=None, - body="", - error=str(exc), - ) - - @staticmethod - def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult: - logger.warning( - "scholar_source.fetch_http_error", - extra={ - "event": "scholar_source.fetch_http_error", - "requested_url": requested_url, - "status_code": exc.code, - }, - ) - return FetchResult( - requested_url=requested_url, - status_code=exc.code, - final_url=exc.geturl(), - body=LiveScholarSource._http_error_body(exc), - error=str(exc), - ) - - @staticmethod - def _success_result(requested_url: str, response) -> FetchResult: - body = response.read().decode("utf-8", errors="replace") - status_code = getattr(response, "status", 200) - logger.debug( - "scholar_source.fetch_succeeded", - extra={ - "event": "scholar_source.fetch_succeeded", - "requested_url": requested_url, - "status_code": status_code, - }, - ) - return FetchResult( - requested_url=requested_url, - status_code=status_code, - final_url=response.geturl(), - body=body, - error=None, - ) - - def _fetch_sync(self, requested_url: str) -> FetchResult: - request = self._build_request(requested_url) - - try: - with urlopen(request, timeout=self._timeout_seconds) as response: - return self._success_result(requested_url, response) - except HTTPError as exc: - return self._http_error_result(requested_url, exc) - except URLError as exc: - return self._network_error_result(requested_url, exc) - - -def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str: - query: dict[str, int | str] = {"hl": "en", "user": scholar_id} - if cstart > 0: - query["cstart"] = int(cstart) - if pagesize > 0: - query["pagesize"] = int(pagesize) - return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}" - - -def _build_author_search_url(*, query: str, start: int) -> str: - params: dict[str, int | str] = { - "hl": "en", - "view_op": "search_authors", - "mauthors": query, - } - if start > 0: - params["astart"] = int(start) - return f"{SCHOLAR_PROFILE_URL}?{urlencode(params)}" +from app.services.domains.scholar.source import * +from app.services.domains.scholar.source import _build_profile_url diff --git a/app/services/scholars.py b/app/services/scholars.py index dd99e0c..ebba422 100644 --- a/app/services/scholars.py +++ b/app/services/scholars.py @@ -1,1123 +1,3 @@ from __future__ import annotations -import asyncio -from dataclasses import replace -from datetime import datetime -from datetime import timedelta -from datetime import timezone -import logging -import os -import random -import re -from pathlib import Path -from urllib.parse import urlparse -from uuid import uuid4 - -from sqlalchemy import delete, func, select, text -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile -from app.services.scholar_parser import ( - ParseState, - ParsedAuthorSearchPage, - parse_author_search_page, - parse_profile_page, -) -from app.services.scholar_source import ScholarSource - -SCHOLAR_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{12}$") -MAX_IMAGE_URL_LENGTH = 2048 -MAX_AUTHOR_SEARCH_LIMIT = 25 -DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES = 512 -DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS = 300 -DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD = 1 -DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS = 1800 -DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0 -DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0 -DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2 -DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3 -AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global" -AUTHOR_SEARCH_LOCK_NAMESPACE = 3901 -AUTHOR_SEARCH_LOCK_KEY = 1 -ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = { - "image/jpeg": ".jpg", - "image/png": ".png", - "image/webp": ".webp", - "image/gif": ".gif", -} -SEARCH_DISABLED_REASON = "search_disabled_by_configuration" -SEARCH_COOLDOWN_REASON = "search_temporarily_disabled_due_to_repeated_blocks" -SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_response" - -STATE_REASON_HINTS: dict[str, str] = { - SEARCH_DISABLED_REASON: ( - "Scholar name search is currently disabled by service policy. " - "Add scholars by profile URL or Scholar ID." - ), - SEARCH_COOLDOWN_REASON: ( - "Scholar name search is temporarily paused after repeated block responses. " - "Use Scholar URL/ID adds until cooldown expires." - ), - SEARCH_CACHED_BLOCK_REASON: ( - "A recent blocked response was cached to reduce traffic. " - "Retry later or add by Scholar URL/ID." - ), - "network_dns_resolution_failed": ( - "DNS resolution failed while reaching scholar.google.com. " - "Verify container DNS/network and retry." - ), - "network_timeout": ( - "Request timed out before Google Scholar responded. " - "Increase delay/backoff and retry." - ), - "network_tls_error": ( - "TLS handshake/certificate validation failed. " - "Verify outbound TLS/network configuration." - ), - "blocked_http_429_rate_limited": ( - "Google Scholar rate-limited the request. " - "Slow request cadence and retry later." - ), - "blocked_unusual_traffic_detected": ( - "Google Scholar flagged traffic as unusual. " - "Increase delay/jitter and reduce concurrent scraping." - ), - "blocked_accounts_redirect": ( - "Request was redirected to Google Account sign-in. " - "Treat as access block and retry later." - ), -} -logger = logging.getLogger(__name__) - - -class ScholarServiceError(ValueError): - """Raised for expected scholar-management validation failures.""" - - -def validate_scholar_id(value: str) -> str: - scholar_id = value.strip() - if not SCHOLAR_ID_PATTERN.fullmatch(scholar_id): - raise ScholarServiceError("Scholar ID must match [a-zA-Z0-9_-]{12}.") - return scholar_id - - -def normalize_display_name(value: str) -> str | None: - normalized = value.strip() - return normalized if normalized else None - - -def normalize_profile_image_url(value: str | None) -> str | None: - if value is None: - return None - - candidate = value.strip() - if not candidate: - return None - - if len(candidate) > MAX_IMAGE_URL_LENGTH: - raise ScholarServiceError( - f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer." - ) - - parsed = urlparse(candidate) - if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc: - raise ScholarServiceError("Image URL must be an absolute http(s) URL.") - - return candidate - - -def _ensure_upload_root(upload_dir: str, *, create: bool) -> Path: - root = Path(upload_dir).expanduser().resolve() - if create: - root.mkdir(parents=True, exist_ok=True) - return root - - -def _resolve_upload_path(upload_root: Path, relative_path: str) -> Path: - candidate = (upload_root / relative_path).resolve() - if upload_root != candidate and upload_root not in candidate.parents: - raise ScholarServiceError("Invalid scholar image path.") - return candidate - - -def _safe_remove_upload(upload_root: Path, relative_path: str | None) -> None: - if not relative_path: - return - try: - file_path = _resolve_upload_path(upload_root, relative_path) - except ScholarServiceError: - return - - try: - if file_path.exists() and file_path.is_file(): - file_path.unlink() - except OSError: - return - - -def resolve_profile_image( - profile: ScholarProfile, - *, - uploaded_image_url: str | None, -) -> tuple[str | None, str]: - if profile.profile_image_upload_path and uploaded_image_url: - return uploaded_image_url, "upload" - if profile.profile_image_override_url: - return profile.profile_image_override_url, "override" - if profile.profile_image_url: - return profile.profile_image_url, "scraped" - return None, "none" - - -def resolve_upload_file_path(*, upload_dir: str, relative_path: str) -> Path: - root = _ensure_upload_root(upload_dir, create=False) - return _resolve_upload_path(root, relative_path) - - -def scrape_state_hint(*, state: ParseState, state_reason: str) -> str | None: - if state not in {ParseState.NETWORK_ERROR, ParseState.BLOCKED_OR_CAPTCHA}: - return None - return STATE_REASON_HINTS.get(state_reason) - - -def _merge_warnings(base: list[str], extra: list[str]) -> list[str]: - if not extra: - return sorted(set(base)) - return sorted(set(base + extra)) - - -def _trim_author_search_result( - parsed: ParsedAuthorSearchPage, - *, - limit: int, - extra_warnings: list[str] | None = None, - state_reason_override: str | None = None, -) -> ParsedAuthorSearchPage: - return ParsedAuthorSearchPage( - state=parsed.state, - state_reason=state_reason_override or parsed.state_reason, - candidates=parsed.candidates[: max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT))], - marker_counts=parsed.marker_counts, - warnings=_merge_warnings(parsed.warnings, extra_warnings or []), - ) - - -def _policy_blocked_author_search_result( - *, - reason: str, - warning_codes: list[str], - limit: int, -) -> ParsedAuthorSearchPage: - _ = limit - return ParsedAuthorSearchPage( - state=ParseState.BLOCKED_OR_CAPTCHA, - state_reason=reason, - candidates=[], - marker_counts={}, - warnings=_merge_warnings([], warning_codes), - ) - - -async def _acquire_author_search_lock(db_session: AsyncSession) -> None: - await db_session.execute( - text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"), - { - "namespace": AUTHOR_SEARCH_LOCK_NAMESPACE, - "lock_key": AUTHOR_SEARCH_LOCK_KEY, - }, - ) - - -async def _load_runtime_state_for_update( - db_session: AsyncSession, -) -> AuthorSearchRuntimeState: - result = await db_session.execute( - select(AuthorSearchRuntimeState) - .where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY) - .with_for_update() - ) - state = result.scalar_one_or_none() - if state is not None: - return state - - state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY) - db_session.add(state) - await db_session.flush() - return state - - -def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict: - return { - "state": parsed.state.value, - "state_reason": parsed.state_reason, - "marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()}, - "warnings": [str(value) for value in parsed.warnings], - "candidates": [ - { - "scholar_id": candidate.scholar_id, - "display_name": candidate.display_name, - "affiliation": candidate.affiliation, - "email_domain": candidate.email_domain, - "cited_by_count": candidate.cited_by_count, - "interests": [str(interest) for interest in candidate.interests], - "profile_url": candidate.profile_url, - "profile_image_url": candidate.profile_image_url, - } - for candidate in parsed.candidates - ], - } - - -def _payload_state(payload: dict[str, object]) -> ParseState | None: - state_raw = str(payload.get("state", "")).strip() - try: - return ParseState(state_raw) - except ValueError: - return None - - -def _payload_marker_counts(payload: dict[str, object]) -> dict[str, int]: - marker_counts_payload = payload.get("marker_counts") - if not isinstance(marker_counts_payload, dict): - return {} - parsed: dict[str, int] = {} - for key, value in marker_counts_payload.items(): - try: - parsed[str(key)] = int(value) - except (TypeError, ValueError): - continue - return parsed - - -def _payload_warnings(payload: dict[str, object]) -> list[str]: - warnings_payload = payload.get("warnings") - if not isinstance(warnings_payload, list): - return [] - return [str(value) for value in warnings_payload if isinstance(value, str)] - - -def _parse_optional_string(value: object) -> str | None: - if value is None: - return None - normalized = str(value).strip() - return normalized or None - - -def _parse_optional_int(value: object) -> int | None: - if isinstance(value, int): - return value - if isinstance(value, str) and value.strip(): - try: - return int(value) - except ValueError: - return None - return None - - -def _normalize_interests(value: object) -> list[str]: - if not isinstance(value, list): - return [] - return [str(item) for item in value if isinstance(item, str)] - - -def _deserialize_candidate_payload(value: object) -> dict[str, object] | None: - if not isinstance(value, dict): - return None - scholar_id = str(value.get("scholar_id", "")).strip() - display_name = str(value.get("display_name", "")).strip() - profile_url = str(value.get("profile_url", "")).strip() - if not scholar_id or not display_name or not profile_url: - return None - return { - "scholar_id": scholar_id, - "display_name": display_name, - "affiliation": _parse_optional_string(value.get("affiliation")), - "email_domain": _parse_optional_string(value.get("email_domain")), - "cited_by_count": _parse_optional_int(value.get("cited_by_count")), - "interests": _normalize_interests(value.get("interests")), - "profile_url": profile_url, - "profile_image_url": _parse_optional_string(value.get("profile_image_url")), - } - - -def _deserialize_candidates(payload: dict[str, object]) -> list[dict[str, object]]: - candidates_payload = payload.get("candidates") - if not isinstance(candidates_payload, list): - return [] - normalized: list[dict[str, object]] = [] - for value in candidates_payload: - candidate = _deserialize_candidate_payload(value) - if candidate is not None: - normalized.append(candidate) - return normalized - - -def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None: - if not isinstance(payload, dict): - return None - - state = _payload_state(payload) - if state is None: - return None - - marker_counts = _payload_marker_counts(payload) - warnings = _payload_warnings(payload) - from app.services.scholar_parser import ScholarSearchCandidate - - normalized_candidates = _deserialize_candidates(payload) - - return ParsedAuthorSearchPage( - state=state, - state_reason=str(payload.get("state_reason", "")).strip() or "unknown", - candidates=[ - ScholarSearchCandidate( - scholar_id=item["scholar_id"], - display_name=item["display_name"], - affiliation=item["affiliation"], - email_domain=item["email_domain"], - cited_by_count=item["cited_by_count"], - interests=item["interests"], - profile_url=item["profile_url"], - profile_image_url=item["profile_image_url"], - ) - for item in normalized_candidates - ], - marker_counts=marker_counts, - warnings=warnings, - ) - - -async def _cache_get_author_search_result( - db_session: AsyncSession, - *, - query_key: str, - now_utc: datetime, -) -> ParsedAuthorSearchPage | None: - result = await db_session.execute( - select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) - ) - entry = result.scalar_one_or_none() - if entry is None: - return None - expires_at = entry.expires_at - if expires_at.tzinfo is None: - expires_at = expires_at.replace(tzinfo=timezone.utc) - if expires_at <= now_utc: - await db_session.delete(entry) - return None - parsed = _deserialize_parsed_author_search_page(entry.payload) - if parsed is None: - await db_session.delete(entry) - return None - return parsed - - -async def _cache_set_author_search_result( - db_session: AsyncSession, - *, - query_key: str, - parsed: ParsedAuthorSearchPage, - ttl_seconds: float, - max_entries: int, - now_utc: datetime, -) -> None: - ttl = max(float(ttl_seconds), 0.0) - existing_result = await db_session.execute( - select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key) - ) - existing = existing_result.scalar_one_or_none() - - if ttl <= 0.0: - if existing is not None: - await db_session.delete(existing) - return - - expires_at = now_utc + timedelta(seconds=ttl) - payload = _serialize_parsed_author_search_page(parsed) - if existing is None: - db_session.add( - AuthorSearchCacheEntry( - query_key=query_key, - payload=payload, - expires_at=expires_at, - cached_at=now_utc, - updated_at=now_utc, - ) - ) - else: - existing.payload = payload - existing.expires_at = expires_at - existing.cached_at = now_utc - existing.updated_at = now_utc - - await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries) - - -async def _prune_author_search_cache( - db_session: AsyncSession, - *, - now_utc: datetime, - max_entries: int, -) -> None: - await db_session.execute( - delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc) - ) - bounded_max_entries = max(1, int(max_entries)) - count_result = await db_session.execute( - select(func.count()).select_from(AuthorSearchCacheEntry) - ) - entry_count = int(count_result.scalar_one() or 0) - overflow = max(0, entry_count - bounded_max_entries) - if overflow <= 0: - return - stale_keys_result = await db_session.execute( - select(AuthorSearchCacheEntry.query_key) - .order_by(AuthorSearchCacheEntry.cached_at.asc()) - .limit(overflow) - ) - stale_keys = [str(row[0]) for row in stale_keys_result.all()] - if stale_keys: - await db_session.execute( - delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys)) - ) - - -def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool: - return parsed.state == ParseState.BLOCKED_OR_CAPTCHA - - -def _author_search_cooldown_remaining_seconds( - runtime_state: AuthorSearchRuntimeState, - now_utc: datetime, -) -> int: - cooldown_until = runtime_state.cooldown_until - if cooldown_until is None: - return 0 - if cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - remaining_seconds = int((cooldown_until - now_utc).total_seconds()) - return max(0, remaining_seconds) - - -def _reset_author_search_runtime_state_for_tests() -> None: - # Runtime state now lives in the database; tests should reset via DB fixtures. - return None - - -async def list_scholars_for_user( - db_session: AsyncSession, - *, - user_id: int, -) -> list[ScholarProfile]: - result = await db_session.execute( - select(ScholarProfile) - .where(ScholarProfile.user_id == user_id) - .order_by(ScholarProfile.created_at.desc(), ScholarProfile.id.desc()) - ) - return list(result.scalars().all()) - - -async def create_scholar_for_user( - db_session: AsyncSession, - *, - user_id: int, - scholar_id: str, - display_name: str, - profile_image_url: str | None = None, -) -> ScholarProfile: - profile = ScholarProfile( - user_id=user_id, - scholar_id=validate_scholar_id(scholar_id), - display_name=normalize_display_name(display_name), - profile_image_url=normalize_profile_image_url(profile_image_url), - ) - db_session.add(profile) - try: - await db_session.commit() - except IntegrityError as exc: - await db_session.rollback() - raise ScholarServiceError("That scholar is already tracked for this account.") from exc - await db_session.refresh(profile) - return profile - - -async def get_user_scholar_by_id( - db_session: AsyncSession, - *, - user_id: int, - scholar_profile_id: int, -) -> ScholarProfile | None: - result = await db_session.execute( - select(ScholarProfile).where( - ScholarProfile.id == scholar_profile_id, - ScholarProfile.user_id == user_id, - ) - ) - return result.scalar_one_or_none() - - -async def toggle_scholar_enabled( - db_session: AsyncSession, - *, - profile: ScholarProfile, -) -> ScholarProfile: - profile.is_enabled = not profile.is_enabled - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def delete_scholar( - db_session: AsyncSession, - *, - profile: ScholarProfile, - upload_dir: str | None = None, -) -> None: - if upload_dir: - upload_root = _ensure_upload_root(upload_dir, create=True) - _safe_remove_upload(upload_root, profile.profile_image_upload_path) - - await db_session.delete(profile) - await db_session.commit() - - -def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, str]: - normalized_query = query.strip() - if len(normalized_query) < 2: - raise ScholarServiceError("Search query must be at least 2 characters.") - bounded_limit = max(1, min(int(limit), MAX_AUTHOR_SEARCH_LIMIT)) - return normalized_query, bounded_limit, normalized_query.casefold() - - -def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage: - logger.warning( - "scholar_search.disabled_by_configuration", - extra={"event": "scholar_search.disabled_by_configuration", "query": normalized_query}, - ) - return _policy_blocked_author_search_result( - reason=SEARCH_DISABLED_REASON, - warning_codes=["author_search_disabled_by_configuration"], - limit=bounded_limit, - ) - - -def _normalize_runtime_cooldown_state( - runtime_state: AuthorSearchRuntimeState, - *, - now_utc: datetime, -) -> bool: - if runtime_state.cooldown_until is None: - return False - cooldown_until = runtime_state.cooldown_until - updated = False - if cooldown_until.tzinfo is None: - cooldown_until = cooldown_until.replace(tzinfo=timezone.utc) - runtime_state.cooldown_until = cooldown_until - updated = True - if now_utc < cooldown_until: - return updated - logger.info( - "scholar_search.cooldown_expired", - extra={"event": "scholar_search.cooldown_expired", "cooldown_until_utc": cooldown_until.isoformat()}, - ) - runtime_state.cooldown_until = None - runtime_state.cooldown_rejection_count = 0 - runtime_state.cooldown_alert_emitted = False - return True - - -def _cooldown_warning_codes( - *, - runtime_state: AuthorSearchRuntimeState, - cooldown_remaining_seconds: int, -) -> list[str]: - warning_codes = [ - "author_search_cooldown_active", - f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s", - ] - if bool(runtime_state.cooldown_alert_emitted): - warning_codes.append("author_search_cooldown_alert_threshold_exceeded") - return warning_codes - - -def _emit_cooldown_threshold_alert( - *, - runtime_state: AuthorSearchRuntimeState, - normalized_query: str, - cooldown_rejection_alert_threshold: int, -) -> bool: - runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1 - threshold = max(1, int(cooldown_rejection_alert_threshold)) - if int(runtime_state.cooldown_rejection_count) < threshold: - return True - if bool(runtime_state.cooldown_alert_emitted): - return True - logger.error( - "scholar_search.cooldown_rejection_threshold_exceeded", - extra={ - "event": "scholar_search.cooldown_rejection_threshold_exceeded", - "query": normalized_query, - "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count), - "threshold": threshold, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, - }, - ) - runtime_state.cooldown_alert_emitted = True - return True - - -def _cooldown_block_result( - *, - runtime_state: AuthorSearchRuntimeState, - normalized_query: str, - bounded_limit: int, - cooldown_rejection_alert_threshold: int, - cooldown_remaining_seconds: int, -) -> ParsedAuthorSearchPage: - _emit_cooldown_threshold_alert( - runtime_state=runtime_state, - normalized_query=normalized_query, - cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, - ) - logger.warning( - "scholar_search.cooldown_active", - extra={ - "event": "scholar_search.cooldown_active", - "query": normalized_query, - "cooldown_remaining_seconds": cooldown_remaining_seconds, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, - }, - ) - return _policy_blocked_author_search_result( - reason=SEARCH_COOLDOWN_REASON, - warning_codes=_cooldown_warning_codes( - runtime_state=runtime_state, - cooldown_remaining_seconds=cooldown_remaining_seconds, - ), - limit=bounded_limit, - ) - - -async def _cache_hit_result( - db_session: AsyncSession, - *, - query_key: str, - now_utc: datetime, - normalized_query: str, - bounded_limit: int, -) -> ParsedAuthorSearchPage | None: - cached = await _cache_get_author_search_result( - db_session, - query_key=query_key, - now_utc=now_utc, - ) - if cached is None: - return None - logger.info( - "scholar_search.cache_hit", - extra={ - "event": "scholar_search.cache_hit", - "query": normalized_query, - "state": cached.state.value, - "state_reason": cached.state_reason, - }, - ) - state_reason_override = SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None - return _trim_author_search_result( - cached, - limit=bounded_limit, - extra_warnings=["author_search_served_from_cache"], - state_reason_override=state_reason_override, - ) - - -def _throttle_sleep_seconds( - *, - runtime_state: AuthorSearchRuntimeState, - now_utc: datetime, - min_interval_seconds: float, - interval_jitter_seconds: float, -) -> tuple[float, bool]: - updated = False - if runtime_state.last_live_request_at is None: - enforced_wait_seconds = 0.0 - else: - last_live_request_at = runtime_state.last_live_request_at - if last_live_request_at.tzinfo is None: - last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc) - runtime_state.last_live_request_at = last_live_request_at - updated = True - enforced_wait_seconds = ( - last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc - ).total_seconds() - jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0)) - return max(0.0, float(enforced_wait_seconds)) + jitter_seconds, updated - - -async def _wait_for_author_search_throttle( - *, - runtime_state: AuthorSearchRuntimeState, - normalized_query: str, - now_utc: datetime, - min_interval_seconds: float, - interval_jitter_seconds: float, -) -> bool: - sleep_seconds, updated = _throttle_sleep_seconds( - runtime_state=runtime_state, - now_utc=now_utc, - min_interval_seconds=min_interval_seconds, - interval_jitter_seconds=interval_jitter_seconds, - ) - if sleep_seconds <= 0.0: - return updated - logger.info( - "scholar_search.throttle_wait", - extra={"event": "scholar_search.throttle_wait", "query": normalized_query, "sleep_seconds": round(sleep_seconds, 3)}, - ) - await asyncio.sleep(sleep_seconds) - return True - - -async def _fetch_author_search_with_retries( - *, - source: ScholarSource, - normalized_query: str, - network_error_retries: int, - retry_backoff_seconds: float, -) -> tuple[ParsedAuthorSearchPage, int, list[str]]: - max_attempts = max(1, int(network_error_retries) + 1) - parsed: ParsedAuthorSearchPage | None = None - retry_warnings: list[str] = [] - retry_scheduled_count = 0 - for attempt_index in range(max_attempts): - fetch_result = await source.fetch_author_search_html(normalized_query, start=0) - parsed = parse_author_search_page(fetch_result) - if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1: - break - retry_warnings.append("network_retry_scheduled_for_author_search") - retry_scheduled_count += 1 - retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index) - if retry_sleep_seconds > 0: - await asyncio.sleep(retry_sleep_seconds) - if parsed is None: - raise ScholarServiceError("Unable to complete scholar author search.") - return parsed, retry_scheduled_count, retry_warnings - - -def _with_retry_warnings( - parsed: ParsedAuthorSearchPage, - *, - retry_warnings: list[str], - retry_scheduled_count: int, - retry_alert_threshold: int, - normalized_query: str, -) -> ParsedAuthorSearchPage: - merged = replace(parsed, warnings=_merge_warnings(parsed.warnings, retry_warnings)) - threshold = max(1, int(retry_alert_threshold)) - if retry_scheduled_count < threshold: - return merged - logger.warning( - "scholar_search.retry_threshold_exceeded", - extra={ - "event": "scholar_search.retry_threshold_exceeded", - "query": normalized_query, - "retry_scheduled_count": retry_scheduled_count, - "threshold": threshold, - "final_state": merged.state.value, - "final_state_reason": merged.state_reason, - }, - ) - return replace( - merged, - warnings=_merge_warnings( - merged.warnings, - [f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"], - ), - ) - - -def _apply_block_circuit_breaker( - *, - runtime_state: AuthorSearchRuntimeState, - merged_parsed: ParsedAuthorSearchPage, - cooldown_block_threshold: int, - cooldown_seconds: int, - normalized_query: str, -) -> ParsedAuthorSearchPage: - if not _is_author_search_block_state(merged_parsed): - runtime_state.consecutive_blocked_count = 0 - return merged_parsed - runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1 - logger.warning( - "scholar_search.block_detected", - extra={ - "event": "scholar_search.block_detected", - "query": normalized_query, - "state_reason": merged_parsed.state_reason, - "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count), - }, - ) - if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)): - return merged_parsed - runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds))) - runtime_state.consecutive_blocked_count = 0 - runtime_state.cooldown_rejection_count = 0 - runtime_state.cooldown_alert_emitted = False - logger.error( - "scholar_search.cooldown_activated", - extra={ - "event": "scholar_search.cooldown_activated", - "query": normalized_query, - "cooldown_until_utc": runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None, - }, - ) - return replace( - merged_parsed, - warnings=_merge_warnings(merged_parsed.warnings, ["author_search_circuit_breaker_armed"]), - ) - - -def _resolve_author_search_cache_ttl_seconds( - *, - merged_parsed: ParsedAuthorSearchPage, - blocked_cache_ttl_seconds: int, - cache_ttl_seconds: int, -) -> int: - if _is_author_search_block_state(merged_parsed): - return min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds))) - return max(1, int(cache_ttl_seconds)) - - -async def _load_locked_runtime_state( - db_session: AsyncSession, -) -> AuthorSearchRuntimeState: - await _acquire_author_search_lock(db_session) - return await _load_runtime_state_for_update(db_session) - - -async def _cooldown_or_cache_result( - db_session: AsyncSession, - *, - runtime_state: AuthorSearchRuntimeState, - query_key: str, - normalized_query: str, - bounded_limit: int, - cooldown_rejection_alert_threshold: int, -) -> tuple[ParsedAuthorSearchPage | None, bool]: - runtime_state_updated = _normalize_runtime_cooldown_state( - runtime_state, - now_utc=datetime.now(timezone.utc), - ) - cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds( - runtime_state, - datetime.now(timezone.utc), - ) - if cooldown_remaining_seconds > 0: - return ( - _cooldown_block_result( - runtime_state=runtime_state, - normalized_query=normalized_query, - bounded_limit=bounded_limit, - cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, - cooldown_remaining_seconds=cooldown_remaining_seconds, - ), - True, - ) - cached_result = await _cache_hit_result( - db_session, - query_key=query_key, - now_utc=datetime.now(timezone.utc), - normalized_query=normalized_query, - bounded_limit=bounded_limit, - ) - return cached_result, runtime_state_updated - - -async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]: - runtime_state_updated = await _wait_for_author_search_throttle( - runtime_state=runtime_state, - normalized_query=normalized_query, - now_utc=datetime.now(timezone.utc), - min_interval_seconds=min_interval_seconds, - interval_jitter_seconds=interval_jitter_seconds, - ) - parsed, retry_count, retry_warnings = await _fetch_author_search_with_retries( - source=source, - normalized_query=normalized_query, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - runtime_state.last_live_request_at = datetime.now(timezone.utc) - merged = _with_retry_warnings( - parsed, - retry_warnings=retry_warnings, - retry_scheduled_count=retry_count, - retry_alert_threshold=retry_alert_threshold, - normalized_query=normalized_query, - ) - merged = _apply_block_circuit_breaker( - runtime_state=runtime_state, - merged_parsed=merged, - cooldown_block_threshold=cooldown_block_threshold, - cooldown_seconds=cooldown_seconds, - normalized_query=normalized_query, - ) - ttl_seconds = _resolve_author_search_cache_ttl_seconds( - merged_parsed=merged, - blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, - cache_ttl_seconds=cache_ttl_seconds, - ) - await _cache_set_author_search_result( - db_session, - query_key=query_key, - parsed=merged, - ttl_seconds=float(ttl_seconds), - max_entries=cache_max_entries, - now_utc=datetime.now(timezone.utc), - ) - return merged, True - - -async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage: - normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit) - if not search_enabled: - return _disabled_search_result( - normalized_query=normalized_query, - bounded_limit=bounded_limit, - ) - - runtime_state = await _load_locked_runtime_state(db_session) - early_result, runtime_state_updated = await _cooldown_or_cache_result( - db_session, - runtime_state=runtime_state, - query_key=query_key, - normalized_query=normalized_query, - bounded_limit=bounded_limit, - cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold, - ) - if early_result is not None: - await db_session.commit() - return early_result - - merged_parsed, live_runtime_updated = await _perform_live_author_search( - db_session, - source=source, - runtime_state=runtime_state, - normalized_query=normalized_query, - query_key=query_key, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - min_interval_seconds=min_interval_seconds, - interval_jitter_seconds=interval_jitter_seconds, - retry_alert_threshold=retry_alert_threshold, - cooldown_block_threshold=cooldown_block_threshold, - cooldown_seconds=cooldown_seconds, - blocked_cache_ttl_seconds=blocked_cache_ttl_seconds, - cache_ttl_seconds=cache_ttl_seconds, - cache_max_entries=cache_max_entries, - ) - runtime_state_updated = runtime_state_updated or live_runtime_updated - if runtime_state_updated: - await db_session.commit() - return _trim_author_search_result(merged_parsed, limit=bounded_limit) - - -async def hydrate_profile_metadata( - db_session: AsyncSession, - *, - profile: ScholarProfile, - source: ScholarSource, -) -> ScholarProfile: - fetch_result = await source.fetch_profile_html(profile.scholar_id) - parsed_page = parse_profile_page(fetch_result) - - if parsed_page.profile_name and not (profile.display_name or "").strip(): - profile.display_name = parsed_page.profile_name - if parsed_page.profile_image_url and not profile.profile_image_url: - profile.profile_image_url = parsed_page.profile_image_url - - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def set_profile_image_override_url( - db_session: AsyncSession, - *, - profile: ScholarProfile, - image_url: str | None, - upload_dir: str, -) -> ScholarProfile: - upload_root = _ensure_upload_root(upload_dir, create=True) - _safe_remove_upload(upload_root, profile.profile_image_upload_path) - - profile.profile_image_upload_path = None - profile.profile_image_override_url = normalize_profile_image_url(image_url) - - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def clear_profile_image_customization( - db_session: AsyncSession, - *, - profile: ScholarProfile, - upload_dir: str, -) -> ScholarProfile: - upload_root = _ensure_upload_root(upload_dir, create=True) - _safe_remove_upload(upload_root, profile.profile_image_upload_path) - - profile.profile_image_upload_path = None - profile.profile_image_override_url = None - - await db_session.commit() - await db_session.refresh(profile) - return profile - - -async def set_profile_image_upload( - db_session: AsyncSession, - *, - profile: ScholarProfile, - content_type: str | None, - image_bytes: bytes, - upload_dir: str, - max_upload_bytes: int, -) -> ScholarProfile: - normalized_content_type = (content_type or "").strip().lower() - extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type) - if extension is None: - raise ScholarServiceError( - "Unsupported image type. Use JPEG, PNG, WEBP, or GIF." - ) - - if not image_bytes: - raise ScholarServiceError("Uploaded image file is empty.") - - if len(image_bytes) > max_upload_bytes: - raise ScholarServiceError( - f"Uploaded image exceeds {max_upload_bytes} bytes." - ) - - upload_root = _ensure_upload_root(upload_dir, create=True) - user_dir = upload_root / str(profile.user_id) - user_dir.mkdir(parents=True, exist_ok=True) - - filename = f"{profile.id}_{uuid4().hex}{extension}" - relative_path = os.path.join(str(profile.user_id), filename) - absolute_path = _resolve_upload_path(upload_root, relative_path) - absolute_path.write_bytes(image_bytes) - - old_path = profile.profile_image_upload_path - profile.profile_image_upload_path = relative_path - profile.profile_image_override_url = None - - await db_session.commit() - await db_session.refresh(profile) - - if old_path and old_path != relative_path: - _safe_remove_upload(upload_root, old_path) - - return profile +from app.services.domains.scholars.application import * diff --git a/app/services/user_settings.py b/app/services/user_settings.py index 3548e44..c3be1d7 100644 --- a/app/services/user_settings.py +++ b/app/services/user_settings.py @@ -1,146 +1,3 @@ from __future__ import annotations -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import UserSetting - - -class UserSettingsServiceError(ValueError): - """Raised for expected settings-validation failures.""" - - -NAV_PAGE_DASHBOARD = "dashboard" -NAV_PAGE_SCHOLARS = "scholars" -NAV_PAGE_PUBLICATIONS = "publications" -NAV_PAGE_SETTINGS = "settings" -NAV_PAGE_STYLE_GUIDE = "style-guide" -NAV_PAGE_RUNS = "runs" -NAV_PAGE_USERS = "users" - -ALLOWED_NAV_PAGES = ( - NAV_PAGE_DASHBOARD, - NAV_PAGE_SCHOLARS, - NAV_PAGE_PUBLICATIONS, - NAV_PAGE_SETTINGS, - NAV_PAGE_STYLE_GUIDE, - NAV_PAGE_RUNS, - NAV_PAGE_USERS, -) -REQUIRED_NAV_PAGES = ( - NAV_PAGE_DASHBOARD, - NAV_PAGE_SCHOLARS, - NAV_PAGE_SETTINGS, -) -DEFAULT_NAV_VISIBLE_PAGES = list(ALLOWED_NAV_PAGES) -HARD_MIN_RUN_INTERVAL_MINUTES = 15 -HARD_MIN_REQUEST_DELAY_SECONDS = 2 - - -def resolve_run_interval_minimum(configured_minimum: int | None) -> int: - try: - parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_RUN_INTERVAL_MINUTES - except (TypeError, ValueError): - parsed = HARD_MIN_RUN_INTERVAL_MINUTES - return max(HARD_MIN_RUN_INTERVAL_MINUTES, parsed) - - -def resolve_request_delay_minimum(configured_minimum: int | None) -> int: - try: - parsed = int(configured_minimum) if configured_minimum is not None else HARD_MIN_REQUEST_DELAY_SECONDS - except (TypeError, ValueError): - parsed = HARD_MIN_REQUEST_DELAY_SECONDS - return max(HARD_MIN_REQUEST_DELAY_SECONDS, parsed) - - -def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERVAL_MINUTES) -> int: - try: - parsed = int(value) - except ValueError as exc: - raise UserSettingsServiceError("Check interval must be a whole number.") from exc - effective_minimum = resolve_run_interval_minimum(minimum) - if parsed < effective_minimum: - raise UserSettingsServiceError( - f"Check interval must be at least {effective_minimum} minutes." - ) - return parsed - - -def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_DELAY_SECONDS) -> int: - try: - parsed = int(value) - except ValueError as exc: - raise UserSettingsServiceError("Request delay must be a whole number.") from exc - effective_minimum = resolve_request_delay_minimum(minimum) - if parsed < effective_minimum: - raise UserSettingsServiceError( - f"Request delay must be at least {effective_minimum} seconds." - ) - return parsed - - -def parse_nav_visible_pages(value: object) -> list[str]: - if not isinstance(value, list): - raise UserSettingsServiceError("Navigation visibility must be a list of page ids.") - - deduped: list[str] = [] - seen: set[str] = set() - - for raw_page in value: - if not isinstance(raw_page, str): - raise UserSettingsServiceError("Navigation visibility entries must be strings.") - - page_id = raw_page.strip() - if page_id not in ALLOWED_NAV_PAGES: - raise UserSettingsServiceError(f"Unsupported navigation page id: {page_id}") - - if page_id in seen: - continue - - seen.add(page_id) - deduped.append(page_id) - - missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen] - if missing_required: - raise UserSettingsServiceError( - "Dashboard, Scholars, and Settings must remain visible." - ) - - return deduped - - -async def get_or_create_settings( - db_session: AsyncSession, - *, - user_id: int, -) -> UserSetting: - result = await db_session.execute( - select(UserSetting).where(UserSetting.user_id == user_id) - ) - settings = result.scalar_one_or_none() - if settings is not None: - return settings - - settings = UserSetting(user_id=user_id) - db_session.add(settings) - await db_session.commit() - await db_session.refresh(settings) - return settings - - -async def update_settings( - db_session: AsyncSession, - *, - settings: UserSetting, - auto_run_enabled: bool, - run_interval_minutes: int, - request_delay_seconds: int, - nav_visible_pages: list[str], -) -> UserSetting: - settings.auto_run_enabled = auto_run_enabled - settings.run_interval_minutes = run_interval_minutes - settings.request_delay_seconds = request_delay_seconds - settings.nav_visible_pages = nav_visible_pages - await db_session.commit() - await db_session.refresh(settings) - return settings +from app.services.domains.settings.application import * diff --git a/app/services/users.py b/app/services/users.py index 7f019c4..12c56f6 100644 --- a/app/services/users.py +++ b/app/services/users.py @@ -1,98 +1,3 @@ from __future__ import annotations -import re - -from sqlalchemy import select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db.models import User - -EMAIL_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") - - -class UserServiceError(ValueError): - """Raised for expected user-management validation failures.""" - - -def normalize_email(value: str) -> str: - return value.strip().lower() - - -def validate_email(value: str) -> str: - email = normalize_email(value) - if not EMAIL_PATTERN.fullmatch(email): - raise UserServiceError("Enter a valid email address.") - return email - - -def validate_password(value: str) -> str: - password = value.strip() - if len(password) < 8: - raise UserServiceError("Password must be at least 8 characters.") - return password - - -async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None: - result = await db_session.execute(select(User).where(User.id == user_id)) - return result.scalar_one_or_none() - - -async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None: - result = await db_session.execute( - select(User).where(User.email == normalize_email(email)) - ) - return result.scalar_one_or_none() - - -async def list_users(db_session: AsyncSession) -> list[User]: - result = await db_session.execute(select(User).order_by(User.email.asc())) - return list(result.scalars().all()) - - -async def create_user( - db_session: AsyncSession, - *, - email: str, - password_hash: str, - is_admin: bool, -) -> User: - user = User( - email=validate_email(email), - password_hash=password_hash, - is_admin=is_admin, - is_active=True, - ) - db_session.add(user) - try: - await db_session.commit() - except IntegrityError as exc: - await db_session.rollback() - raise UserServiceError("A user with that email already exists.") from exc - await db_session.refresh(user) - return user - - -async def set_user_active( - db_session: AsyncSession, - *, - user: User, - is_active: bool, -) -> User: - user.is_active = is_active - await db_session.commit() - await db_session.refresh(user) - return user - - -async def set_user_password_hash( - db_session: AsyncSession, - *, - user: User, - password_hash: str, -) -> User: - user.password_hash = password_hash - await db_session.commit() - await db_session.refresh(user) - return user - +from app.services.domains.users.application import * diff --git a/frontend/src/app/AppShell.vue b/frontend/src/app/AppShell.vue index bd0ea11..c13af89 100644 --- a/frontend/src/app/AppShell.vue +++ b/frontend/src/app/AppShell.vue @@ -1,5 +1,5 @@