diff --git a/.env.example b/.env.example index e2180ba..17e0363 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,10 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1 SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800 SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2 SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3 +SCHOLAR_HTTP_USER_AGENT= +SCHOLAR_HTTP_ROTATE_USER_AGENT=0 +SCHOLAR_HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.9 +SCHOLAR_HTTP_COOKIE= # ------------------------------ # OA Enrichment + PDF Resolution diff --git a/DECOMPOSITION_SLICES.md b/DECOMPOSITION_SLICES.md index 8e22261..87604ae 100644 --- a/DECOMPOSITION_SLICES.md +++ b/DECOMPOSITION_SLICES.md @@ -26,7 +26,7 @@ This file contains self-contained decomposition prompts ("slices") to bring the |---|-------|--------| | 1 | Schema split | **DONE** | | 2 | Ingestion: pagination + publication upsert | **DONE** | -| 3 | Ingestion: enrichment + scholar processing + run completion | pending | +| 3 | Ingestion: enrichment + scholar processing + run completion | **DONE** | | 4 | Scholars service + PDF queue | pending | | 5 | Routers + scheduler | pending | @@ -63,7 +63,17 @@ Also fixed two external import paths (`portability/normalize.py`, `portability/p --- -## Slice 3: Decompose `app/services/ingestion/application.py` — enrichment, scholar processing, run completion +## Slice 3: Decompose `app/services/ingestion/application.py` — enrichment, scholar processing, run completion — DONE + +Completed. Extracted four logical chunks from `application.py` (2,085 → 635 lines): + +| File | Lines | Contents | +|---|---|---| +| `app/services/ingestion/enrichment.py` | 323 | `EnrichmentRunner` class — OpenAlex enrichment, identifier discovery, dedup sweep | +| `app/services/ingestion/scholar_processing.py` | 614 | `process_scholar`, `run_scholar_iteration`, continuation queue, outcome resolution | +| `app/services/ingestion/run_completion.py` | 417 | `complete_run_for_user`, failure summary, alert summary, safety outcome, progress tracking | + +Also updated two tests (`test_ingestion_arxiv_rate_limit.py`, `test_ingestion_progress_reporting.py`) and one integration test (`test_deferred_enrichment.py`) to import from new modules. **Why:** Three remaining logical chunks. Extracting all three brings `application.py` down to ~500 lines (the orchestration spine). diff --git a/app/api/router.py b/app/api/router.py index a11d98a..1b97d07 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -2,11 +2,12 @@ from __future__ import annotations from fastapi import APIRouter -from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings +from app.api.routers import admin, admin_dbops, admin_settings, auth, publications, runs, scholars, settings router = APIRouter(prefix="/api/v1") router.include_router(auth.router) router.include_router(admin.router) +router.include_router(admin_settings.router) router.include_router(admin_dbops.router) router.include_router(scholars.router) router.include_router(settings.router) diff --git a/app/api/routers/admin_settings.py b/app/api/routers/admin_settings.py new file mode 100644 index 0000000..c909874 --- /dev/null +++ b/app/api/routers/admin_settings.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, Request + +from app.api.deps import get_api_admin_user +from app.api.errors import ApiException +from app.api.responses import success_payload +from app.api.schemas import ( + AdminScholarHttpSettingsEnvelope, + AdminScholarHttpSettingsUpdateRequest, +) +from app.db.models import User +from app.logging_utils import structured_log +from app.settings import settings + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/admin/settings", tags=["api-admin-settings"]) +_MAX_USER_AGENT_LENGTH = 512 +_MAX_ACCEPT_LANGUAGE_LENGTH = 128 +_MAX_COOKIE_LENGTH = 8192 + + +def _normalize_header_value(value: str, *, field_name: str, max_length: int) -> str: + normalized = value.strip() + if len(normalized) > max_length: + raise ApiException( + status_code=400, + code="invalid_admin_setting", + message=f"{field_name} must be {max_length} characters or fewer.", + ) + return normalized + + +def _serialize_scholar_http_settings() -> dict[str, object]: + return { + "user_agent": settings.scholar_http_user_agent, + "rotate_user_agent": bool(settings.scholar_http_rotate_user_agent), + "accept_language": settings.scholar_http_accept_language, + "cookie": settings.scholar_http_cookie, + } + + +def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest) -> None: + user_agent = _normalize_header_value( + payload.user_agent, + field_name="user_agent", + max_length=_MAX_USER_AGENT_LENGTH, + ) + accept_language = _normalize_header_value( + payload.accept_language, + field_name="accept_language", + max_length=_MAX_ACCEPT_LANGUAGE_LENGTH, + ) + cookie = _normalize_header_value( + payload.cookie, + field_name="cookie", + max_length=_MAX_COOKIE_LENGTH, + ) + object.__setattr__(settings, "scholar_http_user_agent", user_agent) + object.__setattr__(settings, "scholar_http_rotate_user_agent", bool(payload.rotate_user_agent)) + object.__setattr__(settings, "scholar_http_accept_language", accept_language) + object.__setattr__(settings, "scholar_http_cookie", cookie) + + +@router.get( + "/scholar-http", + response_model=AdminScholarHttpSettingsEnvelope, +) +async def get_scholar_http_settings( + request: Request, + admin_user: User = Depends(get_api_admin_user), +): + structured_log( + logger, + "info", + "api.admin.settings.scholar_http_read", + admin_user_id=int(admin_user.id), + ) + return success_payload(request, data=_serialize_scholar_http_settings()) + + +@router.put( + "/scholar-http", + response_model=AdminScholarHttpSettingsEnvelope, +) +async def update_scholar_http_settings( + payload: AdminScholarHttpSettingsUpdateRequest, + request: Request, + admin_user: User = Depends(get_api_admin_user), +): + _apply_scholar_http_settings(payload) + structured_log( + logger, + "info", + "api.admin.settings.scholar_http_updated", + admin_user_id=int(admin_user.id), + rotate_user_agent=bool(settings.scholar_http_rotate_user_agent), + user_agent_set=bool(settings.scholar_http_user_agent.strip()), + cookie_set=bool(settings.scholar_http_cookie.strip()), + ) + return success_payload(request, data=_serialize_scholar_http_settings()) diff --git a/app/api/schemas.py b/app/api/schemas.py deleted file mode 100644 index 0b7db35..0000000 --- a/app/api/schemas.py +++ /dev/null @@ -1,945 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator - - -class ApiMeta(BaseModel): - request_id: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class ApiErrorData(BaseModel): - code: str - message: str - details: Any | None = None - - model_config = ConfigDict(extra="forbid") - - -class ApiErrorEnvelope(BaseModel): - error: ApiErrorData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class MessageData(BaseModel): - message: str - - model_config = ConfigDict(extra="forbid") - - -class MessageEnvelope(BaseModel): - data: MessageData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class SessionUserData(BaseModel): - id: int - email: str - is_admin: bool - is_active: bool - - model_config = ConfigDict(extra="forbid") - - -class AuthMeData(BaseModel): - authenticated: bool - csrf_token: str - user: SessionUserData - - model_config = ConfigDict(extra="forbid") - - -class AuthMeEnvelope(BaseModel): - data: AuthMeData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class CsrfBootstrapData(BaseModel): - csrf_token: str - authenticated: bool - - model_config = ConfigDict(extra="forbid") - - -class CsrfBootstrapEnvelope(BaseModel): - data: CsrfBootstrapData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class LoginRequest(BaseModel): - email: str - password: str - - model_config = ConfigDict(extra="forbid") - - -class LoginData(BaseModel): - authenticated: bool - csrf_token: str - user: SessionUserData - - model_config = ConfigDict(extra="forbid") - - -class LoginEnvelope(BaseModel): - data: LoginData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ChangePasswordRequest(BaseModel): - current_password: str - new_password: str - confirm_password: str - - model_config = ConfigDict(extra="forbid") - - -class ScholarItemData(BaseModel): - id: int - scholar_id: str - display_name: str | None - profile_image_url: str | None - profile_image_source: str - is_enabled: bool - baseline_completed: bool - last_run_dt: datetime | None - last_run_status: str | None - - model_config = ConfigDict(extra="forbid") - - -class ScholarsListData(BaseModel): - scholars: list[ScholarItemData] - - model_config = ConfigDict(extra="forbid") - - -class ScholarsListEnvelope(BaseModel): - data: ScholarsListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ScholarCreateRequest(BaseModel): - scholar_id: str - profile_image_url: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class ScholarEnvelope(BaseModel): - data: ScholarItemData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ScholarSearchCandidateData(BaseModel): - scholar_id: str - display_name: str - affiliation: str | None - email_domain: str | None - cited_by_count: int | None - interests: list[str] = Field(default_factory=list) - profile_url: str - profile_image_url: str | None - - model_config = ConfigDict(extra="forbid") - - -class ScholarSearchData(BaseModel): - query: str - state: str - state_reason: str - action_hint: str | None = None - candidates: list[ScholarSearchCandidateData] - warnings: list[str] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class ScholarSearchEnvelope(BaseModel): - data: ScholarSearchData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ScholarImageUrlUpdateRequest(BaseModel): - image_url: str - - model_config = ConfigDict(extra="forbid") - - -class ScholarExportItemData(BaseModel): - scholar_id: str - display_name: str | None = None - is_enabled: bool = True - profile_image_override_url: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class PublicationExportItemData(BaseModel): - scholar_id: str - cluster_id: str | None = None - fingerprint_sha256: str | None = None - title: str - year: int | None = None - citation_count: int = 0 - author_text: str | None = None - venue_text: str | None = None - pub_url: str | None = None - pdf_url: str | None = None - is_read: bool = False - - model_config = ConfigDict(extra="forbid") - - -class DataExportData(BaseModel): - schema_version: int - exported_at: str - scholars: list[ScholarExportItemData] - publications: list[PublicationExportItemData] - - model_config = ConfigDict(extra="forbid") - - -class DataExportEnvelope(BaseModel): - data: DataExportData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class DataImportRequest(BaseModel): - schema_version: int | None = None - scholars: list[ScholarExportItemData] = Field(default_factory=list) - publications: list[PublicationExportItemData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class DataImportResultData(BaseModel): - scholars_created: int - scholars_updated: int - publications_created: int - publications_updated: int - links_created: int - links_updated: int - skipped_records: int - - model_config = ConfigDict(extra="forbid") - - -class DataImportEnvelope(BaseModel): - data: DataImportResultData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class RunListItemData(BaseModel): - id: int - trigger_type: str - status: str - start_dt: datetime - end_dt: datetime | None - scholar_count: int - new_publication_count: int - failed_count: int - partial_count: int - - model_config = ConfigDict(extra="forbid") - - -class RunsListData(BaseModel): - runs: list[RunListItemData] - safety_state: ScrapeSafetyStateData - - model_config = ConfigDict(extra="forbid") - - -class RunsListEnvelope(BaseModel): - data: RunsListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class RunSummaryData(BaseModel): - succeeded_count: int - failed_count: int - partial_count: int - failed_state_counts: dict[str, int] = Field(default_factory=dict) - failed_reason_counts: dict[str, int] = Field(default_factory=dict) - scrape_failure_counts: dict[str, int] = Field(default_factory=dict) - retry_counts: dict[str, int] = Field(default_factory=dict) - alert_thresholds: dict[str, int] = Field(default_factory=dict) - alert_flags: dict[str, bool] = Field(default_factory=dict) - - model_config = ConfigDict(extra="forbid") - - -class ScrapeSafetyCountersData(BaseModel): - consecutive_blocked_runs: int = 0 - consecutive_network_runs: int = 0 - cooldown_entry_count: int = 0 - blocked_start_count: int = 0 - last_blocked_failure_count: int = 0 - last_network_failure_count: int = 0 - last_evaluated_run_id: int | None = None - - model_config = ConfigDict(extra="forbid") - - -class ScrapeSafetyStateData(BaseModel): - cooldown_active: bool - cooldown_reason: str | None = None - cooldown_reason_label: str | None = None - cooldown_until: datetime | None = None - cooldown_remaining_seconds: int = 0 - recommended_action: str | None = None - counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData) - - model_config = ConfigDict(extra="forbid") - - -class RunAttemptLogData(BaseModel): - attempt: int - cstart: int - state: str | None = None - state_reason: str | None = None - status_code: int | None = None - fetch_error: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class RunPageLogData(BaseModel): - page: int - cstart: int - state: str - state_reason: str | None = None - status_code: int | None = None - publication_count: int = 0 - attempt_count: int = 0 - has_show_more_button: bool = False - articles_range: str | None = None - warning_codes: list[str] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class RunDebugData(BaseModel): - status_code: int | None = None - final_url: str | None = None - fetch_error: str | None = None - body_sha256: str | None = None - body_length: int | None = None - has_show_more_button: bool | None = None - articles_range: str | None = None - state_reason: str | None = None - warning_codes: list[str] = Field(default_factory=list) - marker_counts_nonzero: dict[str, int] = Field(default_factory=dict) - page_logs: list[RunPageLogData] = Field(default_factory=list) - attempt_log: list[RunAttemptLogData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class RunScholarResultData(BaseModel): - scholar_profile_id: int - scholar_id: str - state: str - state_reason: str | None = None - outcome: str - attempt_count: int = 0 - publication_count: int = 0 - start_cstart: int = 0 - continuation_cstart: int | None = None - continuation_enqueued: bool = False - continuation_cleared: bool = False - warnings: list[str] = Field(default_factory=list) - error: str | None = None - debug: RunDebugData | None = None - - model_config = ConfigDict(extra="forbid") - - -class RunDetailData(BaseModel): - run: RunListItemData - summary: RunSummaryData - scholar_results: list[RunScholarResultData] - safety_state: ScrapeSafetyStateData - - model_config = ConfigDict(extra="forbid") - - -class RunDetailEnvelope(BaseModel): - data: RunDetailData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class ManualRunData(BaseModel): - run_id: int - status: str - scholar_count: int - succeeded_count: int - failed_count: int - partial_count: int - new_publication_count: int - reused_existing_run: bool - idempotency_key: str | None = None - safety_state: ScrapeSafetyStateData - - model_config = ConfigDict(extra="forbid") - - -class ManualRunEnvelope(BaseModel): - data: ManualRunData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class QueueItemData(BaseModel): - id: int - scholar_profile_id: int - scholar_label: str - status: str - reason: str - dropped_reason: str | None - attempt_count: int - resume_cstart: int - next_attempt_dt: datetime | None - updated_at: datetime - last_error: str | None - last_run_id: int | None - - model_config = ConfigDict(extra="forbid") - - -class QueueListData(BaseModel): - queue_items: list[QueueItemData] - - model_config = ConfigDict(extra="forbid") - - -class QueueListEnvelope(BaseModel): - data: QueueListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class QueueItemEnvelope(BaseModel): - data: QueueItemData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class QueueClearData(BaseModel): - queue_item_id: int - previous_status: str - status: str - message: str - - model_config = ConfigDict(extra="forbid") - - -class QueueClearEnvelope(BaseModel): - data: QueueClearData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminUserData(BaseModel): - id: int - email: str - is_active: bool - is_admin: bool - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(extra="forbid") - - -class AdminUsersListData(BaseModel): - users: list[AdminUserData] - - model_config = ConfigDict(extra="forbid") - - -class AdminUsersListEnvelope(BaseModel): - data: AdminUsersListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminUserCreateRequest(BaseModel): - email: str - password: str - is_admin: bool = False - - model_config = ConfigDict(extra="forbid") - - -class AdminUserActiveUpdateRequest(BaseModel): - is_active: bool - - model_config = ConfigDict(extra="forbid") - - -class AdminUserEnvelope(BaseModel): - data: AdminUserData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminResetPasswordRequest(BaseModel): - new_password: str - - model_config = ConfigDict(extra="forbid") - - -class AdminDbIntegrityCheckData(BaseModel): - name: str - count: int - severity: str - message: str - - model_config = ConfigDict(extra="forbid") - - -class AdminDbIntegrityData(BaseModel): - status: str - checked_at: datetime - failures: list[str] = Field(default_factory=list) - warnings: list[str] = Field(default_factory=list) - checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class AdminDbIntegrityEnvelope(BaseModel): - data: AdminDbIntegrityData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminDbRepairJobData(BaseModel): - id: int - job_name: str - requested_by: str | None - dry_run: bool - status: str - scope: dict[str, Any] = Field(default_factory=dict) - summary: dict[str, Any] = Field(default_factory=dict) - error_text: str | None - started_at: datetime | None - finished_at: datetime | None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(extra="forbid") - - -class AdminDbRepairJobsData(BaseModel): - jobs: list[AdminDbRepairJobData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class AdminDbRepairJobsEnvelope(BaseModel): - data: AdminDbRepairJobsData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class DisplayIdentifierData(BaseModel): - kind: str - value: str - label: str - url: str | None - confidence_score: float = Field(ge=0.0, le=1.0) - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueItemData(BaseModel): - publication_id: int - title: str - display_identifier: DisplayIdentifierData | None = None - pdf_url: str | None - status: str - attempt_count: int - last_failure_reason: str | None - last_failure_detail: str | None - last_source: str | None - requested_by_user_id: int | None - requested_by_email: str | None - queued_at: datetime | None - last_attempt_at: datetime | None - resolved_at: datetime | None - updated_at: datetime - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueData(BaseModel): - items: list[AdminPdfQueueItemData] = Field(default_factory=list) - total_count: int = 0 - page: int = 1 - page_size: int = 100 - has_next: bool = False - has_prev: bool = False - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueEnvelope(BaseModel): - data: AdminPdfQueueData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueRequeueData(BaseModel): - publication_id: int - queued: bool - status: str - message: str - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueRequeueEnvelope(BaseModel): - data: AdminPdfQueueRequeueData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueBulkEnqueueData(BaseModel): - requested_count: int - queued_count: int - message: str - - model_config = ConfigDict(extra="forbid") - - -class AdminPdfQueueBulkEnqueueEnvelope(BaseModel): - data: AdminPdfQueueBulkEnqueueData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminRepairPublicationLinksRequest(BaseModel): - scope_mode: Literal["single_user", "all_users"] = "single_user" - user_id: int | None = Field(default=None, ge=1) - scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200) - dry_run: bool = True - gc_orphan_publications: bool = False - requested_by: str | None = None - confirmation_text: str | None = None - - model_config = ConfigDict(extra="forbid") - - @model_validator(mode="after") - def validate_scope(self) -> AdminRepairPublicationLinksRequest: - if self.scope_mode == "single_user" and self.user_id is None: - raise ValueError("user_id is required when scope_mode=single_user.") - if self.scope_mode == "all_users" and self.user_id is not None: - raise ValueError("user_id must be omitted when scope_mode=all_users.") - if not self.dry_run and self.scope_mode == "all_users": - expected = "REPAIR ALL USERS" - provided = (self.confirmation_text or "").strip() - if provided != expected: - raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.") - return self - - -class AdminRepairPublicationLinksResultData(BaseModel): - job_id: int - status: str - scope: dict[str, Any] = Field(default_factory=dict) - summary: dict[str, Any] = Field(default_factory=dict) - - model_config = ConfigDict(extra="forbid") - - -class AdminRepairPublicationLinksEnvelope(BaseModel): - data: AdminRepairPublicationLinksResultData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class AdminNearDuplicateClusterMemberData(BaseModel): - publication_id: int - title: str - year: int | None - citation_count: int - - model_config = ConfigDict(extra="forbid") - - -class AdminNearDuplicateClusterData(BaseModel): - cluster_key: str - winner_publication_id: int - member_count: int - similarity_score: float = Field(ge=0.0, le=1.0) - members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class AdminRepairPublicationNearDuplicatesRequest(BaseModel): - dry_run: bool = True - similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0) - min_shared_tokens: int = Field(default=3, ge=1, le=8) - max_year_delta: int = Field(default=1, ge=0, le=5) - max_clusters: int = Field(default=25, ge=1, le=200) - selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200) - requested_by: str | None = None - confirmation_text: str | None = None - - model_config = ConfigDict(extra="forbid") - - @model_validator(mode="after") - def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest: - if self.dry_run: - return self - if not self.selected_cluster_keys: - raise ValueError("selected_cluster_keys is required when dry_run=false.") - expected = "MERGE SELECTED DUPLICATES" - provided = (self.confirmation_text or "").strip() - if provided != expected: - raise ValueError( - "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges." - ) - return self - - -class AdminRepairPublicationNearDuplicatesResultData(BaseModel): - job_id: int - status: str - scope: dict[str, Any] = Field(default_factory=dict) - summary: dict[str, Any] = Field(default_factory=dict) - clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list) - - model_config = ConfigDict(extra="forbid") - - -class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel): - data: AdminRepairPublicationNearDuplicatesResultData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class SettingsPolicyData(BaseModel): - min_run_interval_minutes: int - min_request_delay_seconds: int - automation_allowed: bool - manual_run_allowed: bool - blocked_failure_threshold: int - network_failure_threshold: int - cooldown_blocked_seconds: int - cooldown_network_seconds: int - - model_config = ConfigDict(extra="forbid") - - -class SettingsData(BaseModel): - auto_run_enabled: bool - run_interval_minutes: int - request_delay_seconds: int - nav_visible_pages: list[str] - policy: SettingsPolicyData - safety_state: ScrapeSafetyStateData - - openalex_api_key: str | None = None - crossref_api_token: str | None = None - crossref_api_mailto: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class SettingsEnvelope(BaseModel): - data: SettingsData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class SettingsUpdateRequest(BaseModel): - auto_run_enabled: bool - run_interval_minutes: int - request_delay_seconds: int - nav_visible_pages: list[str] | None = None - - openalex_api_key: str | None = None - crossref_api_token: str | None = None - crossref_api_mailto: str | None = None - - model_config = ConfigDict(extra="forbid") - - -class PublicationItemData(BaseModel): - publication_id: int - scholar_profile_id: int - scholar_label: str - title: str - year: int | None - citation_count: int - venue_text: str | None - pub_url: str | None - display_identifier: DisplayIdentifierData | None = None - pdf_url: str | None - pdf_status: str = "untracked" - pdf_attempt_count: int = 0 - pdf_failure_reason: str | None = None - pdf_failure_detail: str | None = None - is_read: bool - is_favorite: bool = False - first_seen_at: datetime - is_new_in_latest_run: bool - - model_config = ConfigDict(extra="forbid") - - -class PublicationsListData(BaseModel): - mode: str - favorite_only: bool = False - selected_scholar_profile_id: int | None - unread_count: int - favorites_count: int - latest_count: int - new_count: int - total_count: int - page: int - page_size: int - snapshot: str - has_next: bool = False - has_prev: bool = False - publications: list[PublicationItemData] - - model_config = ConfigDict(extra="forbid") - - -class PublicationsListEnvelope(BaseModel): - data: PublicationsListData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class MarkAllReadData(BaseModel): - message: str - updated_count: int - - model_config = ConfigDict(extra="forbid") - - -class MarkAllReadEnvelope(BaseModel): - data: MarkAllReadData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class PublicationSelectionItem(BaseModel): - scholar_profile_id: int - publication_id: int - - model_config = ConfigDict(extra="forbid") - - -class MarkSelectedReadRequest(BaseModel): - selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500) - - model_config = ConfigDict(extra="forbid") - - -class MarkSelectedReadData(BaseModel): - message: str - requested_count: int - updated_count: int - - model_config = ConfigDict(extra="forbid") - - -class MarkSelectedReadEnvelope(BaseModel): - data: MarkSelectedReadData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class RetryPublicationPdfRequest(BaseModel): - scholar_profile_id: int = Field(ge=1) - - model_config = ConfigDict(extra="forbid") - - -class RetryPublicationPdfData(BaseModel): - message: str - queued: bool - resolved_pdf: bool - publication: PublicationItemData - - model_config = ConfigDict(extra="forbid") - - -class RetryPublicationPdfEnvelope(BaseModel): - data: RetryPublicationPdfData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") - - -class TogglePublicationFavoriteRequest(BaseModel): - scholar_profile_id: int = Field(ge=1) - is_favorite: bool - - model_config = ConfigDict(extra="forbid") - - -class TogglePublicationFavoriteData(BaseModel): - message: str - publication: PublicationItemData - - model_config = ConfigDict(extra="forbid") - - -class TogglePublicationFavoriteEnvelope(BaseModel): - data: TogglePublicationFavoriteData - meta: ApiMeta - - model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/__init__.py b/app/api/schemas/__init__.py new file mode 100644 index 0000000..4114e70 --- /dev/null +++ b/app/api/schemas/__init__.py @@ -0,0 +1,7 @@ +from app.api.schemas.admin import * # noqa: F401, F403 +from app.api.schemas.auth import * # noqa: F401, F403 +from app.api.schemas.common import * # noqa: F401, F403 +from app.api.schemas.publications import * # noqa: F401, F403 +from app.api.schemas.runs import * # noqa: F401, F403 +from app.api.schemas.scholars import * # noqa: F401, F403 +from app.api.schemas.settings import * # noqa: F401, F403 diff --git a/app/api/schemas/admin.py b/app/api/schemas/admin.py new file mode 100644 index 0000000..d9f2eb7 --- /dev/null +++ b/app/api/schemas/admin.py @@ -0,0 +1,314 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from app.api.schemas.common import ApiMeta +from app.api.schemas.publications import DisplayIdentifierData + + +class AdminUserData(BaseModel): + id: int + email: str + is_active: bool + is_admin: bool + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminUsersListData(BaseModel): + users: list[AdminUserData] + + model_config = ConfigDict(extra="forbid") + + +class AdminUsersListEnvelope(BaseModel): + data: AdminUsersListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminUserCreateRequest(BaseModel): + email: str + password: str + is_admin: bool = False + + model_config = ConfigDict(extra="forbid") + + +class AdminUserActiveUpdateRequest(BaseModel): + is_active: bool + + model_config = ConfigDict(extra="forbid") + + +class AdminUserEnvelope(BaseModel): + data: AdminUserData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminResetPasswordRequest(BaseModel): + new_password: str + + model_config = ConfigDict(extra="forbid") + + +class AdminScholarHttpSettingsData(BaseModel): + user_agent: str + rotate_user_agent: bool + accept_language: str + cookie: str + + model_config = ConfigDict(extra="forbid") + + +class AdminScholarHttpSettingsEnvelope(BaseModel): + data: AdminScholarHttpSettingsData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminScholarHttpSettingsUpdateRequest(BaseModel): + user_agent: str + rotate_user_agent: bool + accept_language: str + cookie: str + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityCheckData(BaseModel): + name: str + count: int + severity: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityData(BaseModel): + status: str + checked_at: datetime + failures: list[str] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminDbIntegrityEnvelope(BaseModel): + data: AdminDbIntegrityData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobData(BaseModel): + id: int + job_name: str + requested_by: str | None + dry_run: bool + status: str + scope: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + error_text: str | None + started_at: datetime | None + finished_at: datetime | None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobsData(BaseModel): + jobs: list[AdminDbRepairJobData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminDbRepairJobsEnvelope(BaseModel): + data: AdminDbRepairJobsData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueItemData(BaseModel): + publication_id: int + title: str + display_identifier: DisplayIdentifierData | None = None + pdf_url: str | None + status: str + attempt_count: int + last_failure_reason: str | None + last_failure_detail: str | None + last_source: str | None + requested_by_user_id: int | None + requested_by_email: str | None + queued_at: datetime | None + last_attempt_at: datetime | None + resolved_at: datetime | None + updated_at: datetime + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueData(BaseModel): + items: list[AdminPdfQueueItemData] = Field(default_factory=list) + total_count: int = 0 + page: int = 1 + page_size: int = 100 + has_next: bool = False + has_prev: bool = False + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueEnvelope(BaseModel): + data: AdminPdfQueueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueRequeueData(BaseModel): + publication_id: int + queued: bool + status: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueRequeueEnvelope(BaseModel): + data: AdminPdfQueueRequeueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueBulkEnqueueData(BaseModel): + requested_count: int + queued_count: int + message: str + + model_config = ConfigDict(extra="forbid") + + +class AdminPdfQueueBulkEnqueueEnvelope(BaseModel): + data: AdminPdfQueueBulkEnqueueData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationLinksRequest(BaseModel): + scope_mode: Literal["single_user", "all_users"] = "single_user" + user_id: int | None = Field(default=None, ge=1) + scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200) + dry_run: bool = True + gc_orphan_publications: bool = False + requested_by: str | None = None + confirmation_text: str | None = None + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def validate_scope(self) -> AdminRepairPublicationLinksRequest: + if self.scope_mode == "single_user" and self.user_id is None: + raise ValueError("user_id is required when scope_mode=single_user.") + if self.scope_mode == "all_users" and self.user_id is not None: + raise ValueError("user_id must be omitted when scope_mode=all_users.") + if not self.dry_run and self.scope_mode == "all_users": + expected = "REPAIR ALL USERS" + provided = (self.confirmation_text or "").strip() + if provided != expected: + raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.") + return self + + +class AdminRepairPublicationLinksResultData(BaseModel): + job_id: int + status: str + scope: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationLinksEnvelope(BaseModel): + data: AdminRepairPublicationLinksResultData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class AdminNearDuplicateClusterMemberData(BaseModel): + publication_id: int + title: str + year: int | None + citation_count: int + + model_config = ConfigDict(extra="forbid") + + +class AdminNearDuplicateClusterData(BaseModel): + cluster_key: str + winner_publication_id: int + member_count: int + similarity_score: float = Field(ge=0.0, le=1.0) + members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationNearDuplicatesRequest(BaseModel): + dry_run: bool = True + similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0) + min_shared_tokens: int = Field(default=3, ge=1, le=8) + max_year_delta: int = Field(default=1, ge=0, le=5) + max_clusters: int = Field(default=25, ge=1, le=200) + selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200) + requested_by: str | None = None + confirmation_text: str | None = None + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest: + if self.dry_run: + return self + if not self.selected_cluster_keys: + raise ValueError("selected_cluster_keys is required when dry_run=false.") + expected = "MERGE SELECTED DUPLICATES" + provided = (self.confirmation_text or "").strip() + if provided != expected: + raise ValueError( + "confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges." + ) + return self + + +class AdminRepairPublicationNearDuplicatesResultData(BaseModel): + job_id: int + status: str + scope: dict[str, Any] = Field(default_factory=dict) + summary: dict[str, Any] = Field(default_factory=dict) + clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel): + data: AdminRepairPublicationNearDuplicatesResultData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/auth.py b/app/api/schemas/auth.py new file mode 100644 index 0000000..174d0b7 --- /dev/null +++ b/app/api/schemas/auth.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from app.api.schemas.common import ApiMeta + + +class SessionUserData(BaseModel): + id: int + email: str + is_admin: bool + is_active: bool + + model_config = ConfigDict(extra="forbid") + + +class AuthMeData(BaseModel): + authenticated: bool + csrf_token: str + user: SessionUserData + + model_config = ConfigDict(extra="forbid") + + +class AuthMeEnvelope(BaseModel): + data: AuthMeData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class CsrfBootstrapData(BaseModel): + csrf_token: str + authenticated: bool + + model_config = ConfigDict(extra="forbid") + + +class CsrfBootstrapEnvelope(BaseModel): + data: CsrfBootstrapData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class LoginRequest(BaseModel): + email: str + password: str + + model_config = ConfigDict(extra="forbid") + + +class LoginData(BaseModel): + authenticated: bool + csrf_token: str + user: SessionUserData + + model_config = ConfigDict(extra="forbid") + + +class LoginEnvelope(BaseModel): + data: LoginData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ChangePasswordRequest(BaseModel): + current_password: str + new_password: str + confirm_password: str + + model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/common.py b/app/api/schemas/common.py new file mode 100644 index 0000000..cc0805f --- /dev/null +++ b/app/api/schemas/common.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict + + +class ApiMeta(BaseModel): + request_id: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class ApiErrorData(BaseModel): + code: str + message: str + details: Any | None = None + + model_config = ConfigDict(extra="forbid") + + +class ApiErrorEnvelope(BaseModel): + error: ApiErrorData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class MessageData(BaseModel): + message: str + + model_config = ConfigDict(extra="forbid") + + +class MessageEnvelope(BaseModel): + data: MessageData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/publications.py b/app/api/schemas/publications.py new file mode 100644 index 0000000..6ae7c2e --- /dev/null +++ b/app/api/schemas/publications.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + +from app.api.schemas.common import ApiMeta + + +class DisplayIdentifierData(BaseModel): + kind: str + value: str + label: str + url: str | None + confidence_score: float = Field(ge=0.0, le=1.0) + + model_config = ConfigDict(extra="forbid") + + +class PublicationItemData(BaseModel): + publication_id: int + scholar_profile_id: int + scholar_label: str + title: str + year: int | None + citation_count: int + venue_text: str | None + pub_url: str | None + display_identifier: DisplayIdentifierData | None = None + pdf_url: str | None + pdf_status: str = "untracked" + pdf_attempt_count: int = 0 + pdf_failure_reason: str | None = None + pdf_failure_detail: str | None = None + is_read: bool + is_favorite: bool = False + first_seen_at: datetime + is_new_in_latest_run: bool + + model_config = ConfigDict(extra="forbid") + + +class PublicationsListData(BaseModel): + mode: str + favorite_only: bool = False + selected_scholar_profile_id: int | None + unread_count: int + favorites_count: int + latest_count: int + new_count: int + total_count: int + page: int + page_size: int + snapshot: str + has_next: bool = False + has_prev: bool = False + publications: list[PublicationItemData] + + model_config = ConfigDict(extra="forbid") + + +class PublicationsListEnvelope(BaseModel): + data: PublicationsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class MarkAllReadData(BaseModel): + message: str + updated_count: int + + model_config = ConfigDict(extra="forbid") + + +class MarkAllReadEnvelope(BaseModel): + data: MarkAllReadData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class PublicationSelectionItem(BaseModel): + scholar_profile_id: int + publication_id: int + + model_config = ConfigDict(extra="forbid") + + +class MarkSelectedReadRequest(BaseModel): + selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500) + + model_config = ConfigDict(extra="forbid") + + +class MarkSelectedReadData(BaseModel): + message: str + requested_count: int + updated_count: int + + model_config = ConfigDict(extra="forbid") + + +class MarkSelectedReadEnvelope(BaseModel): + data: MarkSelectedReadData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class RetryPublicationPdfRequest(BaseModel): + scholar_profile_id: int = Field(ge=1) + + model_config = ConfigDict(extra="forbid") + + +class RetryPublicationPdfData(BaseModel): + message: str + queued: bool + resolved_pdf: bool + publication: PublicationItemData + + model_config = ConfigDict(extra="forbid") + + +class RetryPublicationPdfEnvelope(BaseModel): + data: RetryPublicationPdfData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteRequest(BaseModel): + scholar_profile_id: int = Field(ge=1) + is_favorite: bool + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteData(BaseModel): + message: str + publication: PublicationItemData + + model_config = ConfigDict(extra="forbid") + + +class TogglePublicationFavoriteEnvelope(BaseModel): + data: TogglePublicationFavoriteData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/runs.py b/app/api/schemas/runs.py new file mode 100644 index 0000000..e9e7791 --- /dev/null +++ b/app/api/schemas/runs.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + +from app.api.schemas.common import ApiMeta + + +class RunListItemData(BaseModel): + id: int + trigger_type: str + status: str + start_dt: datetime + end_dt: datetime | None + scholar_count: int + new_publication_count: int + failed_count: int + partial_count: int + + model_config = ConfigDict(extra="forbid") + + +class ScrapeSafetyCountersData(BaseModel): + consecutive_blocked_runs: int = 0 + consecutive_network_runs: int = 0 + cooldown_entry_count: int = 0 + blocked_start_count: int = 0 + last_blocked_failure_count: int = 0 + last_network_failure_count: int = 0 + last_evaluated_run_id: int | None = None + + model_config = ConfigDict(extra="forbid") + + +class ScrapeSafetyStateData(BaseModel): + cooldown_active: bool + cooldown_reason: str | None = None + cooldown_reason_label: str | None = None + cooldown_until: datetime | None = None + cooldown_remaining_seconds: int = 0 + recommended_action: str | None = None + counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData) + + model_config = ConfigDict(extra="forbid") + + +class RunsListData(BaseModel): + runs: list[RunListItemData] + safety_state: ScrapeSafetyStateData + + model_config = ConfigDict(extra="forbid") + + +class RunsListEnvelope(BaseModel): + data: RunsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class RunSummaryData(BaseModel): + succeeded_count: int + failed_count: int + partial_count: int + failed_state_counts: dict[str, int] = Field(default_factory=dict) + failed_reason_counts: dict[str, int] = Field(default_factory=dict) + scrape_failure_counts: dict[str, int] = Field(default_factory=dict) + retry_counts: dict[str, int] = Field(default_factory=dict) + alert_thresholds: dict[str, int] = Field(default_factory=dict) + alert_flags: dict[str, bool] = Field(default_factory=dict) + + model_config = ConfigDict(extra="forbid") + + +class RunAttemptLogData(BaseModel): + attempt: int + cstart: int + state: str | None = None + state_reason: str | None = None + status_code: int | None = None + fetch_error: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class RunPageLogData(BaseModel): + page: int + cstart: int + state: str + state_reason: str | None = None + status_code: int | None = None + publication_count: int = 0 + attempt_count: int = 0 + has_show_more_button: bool = False + articles_range: str | None = None + warning_codes: list[str] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class RunDebugData(BaseModel): + status_code: int | None = None + final_url: str | None = None + fetch_error: str | None = None + body_sha256: str | None = None + body_length: int | None = None + has_show_more_button: bool | None = None + articles_range: str | None = None + state_reason: str | None = None + warning_codes: list[str] = Field(default_factory=list) + marker_counts_nonzero: dict[str, int] = Field(default_factory=dict) + page_logs: list[RunPageLogData] = Field(default_factory=list) + attempt_log: list[RunAttemptLogData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class RunScholarResultData(BaseModel): + scholar_profile_id: int + scholar_id: str + state: str + state_reason: str | None = None + outcome: str + attempt_count: int = 0 + publication_count: int = 0 + start_cstart: int = 0 + continuation_cstart: int | None = None + continuation_enqueued: bool = False + continuation_cleared: bool = False + warnings: list[str] = Field(default_factory=list) + error: str | None = None + debug: RunDebugData | None = None + + model_config = ConfigDict(extra="forbid") + + +class RunDetailData(BaseModel): + run: RunListItemData + summary: RunSummaryData + scholar_results: list[RunScholarResultData] + safety_state: ScrapeSafetyStateData + + model_config = ConfigDict(extra="forbid") + + +class RunDetailEnvelope(BaseModel): + data: RunDetailData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ManualRunData(BaseModel): + run_id: int + status: str + scholar_count: int + succeeded_count: int + failed_count: int + partial_count: int + new_publication_count: int + reused_existing_run: bool + idempotency_key: str | None = None + safety_state: ScrapeSafetyStateData + + model_config = ConfigDict(extra="forbid") + + +class ManualRunEnvelope(BaseModel): + data: ManualRunData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueItemData(BaseModel): + id: int + scholar_profile_id: int + scholar_label: str + status: str + reason: str + dropped_reason: str | None + attempt_count: int + resume_cstart: int + next_attempt_dt: datetime | None + updated_at: datetime + last_error: str | None + last_run_id: int | None + + model_config = ConfigDict(extra="forbid") + + +class QueueListData(BaseModel): + queue_items: list[QueueItemData] + + model_config = ConfigDict(extra="forbid") + + +class QueueListEnvelope(BaseModel): + data: QueueListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueItemEnvelope(BaseModel): + data: QueueItemData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class QueueClearData(BaseModel): + queue_item_id: int + previous_status: str + status: str + message: str + + model_config = ConfigDict(extra="forbid") + + +class QueueClearEnvelope(BaseModel): + data: QueueClearData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/scholars.py b/app/api/schemas/scholars.py new file mode 100644 index 0000000..dda8329 --- /dev/null +++ b/app/api/schemas/scholars.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + +from app.api.schemas.common import ApiMeta + + +class ScholarItemData(BaseModel): + id: int + scholar_id: str + display_name: str | None + profile_image_url: str | None + profile_image_source: str + is_enabled: bool + baseline_completed: bool + last_run_dt: datetime | None + last_run_status: str | None + + model_config = ConfigDict(extra="forbid") + + +class ScholarsListData(BaseModel): + scholars: list[ScholarItemData] + + model_config = ConfigDict(extra="forbid") + + +class ScholarsListEnvelope(BaseModel): + data: ScholarsListData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ScholarCreateRequest(BaseModel): + scholar_id: str + profile_image_url: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class ScholarEnvelope(BaseModel): + data: ScholarItemData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ScholarSearchCandidateData(BaseModel): + scholar_id: str + display_name: str + affiliation: str | None + email_domain: str | None + cited_by_count: int | None + interests: list[str] = Field(default_factory=list) + profile_url: str + profile_image_url: str | None + + model_config = ConfigDict(extra="forbid") + + +class ScholarSearchData(BaseModel): + query: str + state: str + state_reason: str + action_hint: str | None = None + candidates: list[ScholarSearchCandidateData] + warnings: list[str] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class ScholarSearchEnvelope(BaseModel): + data: ScholarSearchData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class ScholarImageUrlUpdateRequest(BaseModel): + image_url: str + + model_config = ConfigDict(extra="forbid") + + +class ScholarExportItemData(BaseModel): + scholar_id: str + display_name: str | None = None + is_enabled: bool = True + profile_image_override_url: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class PublicationExportItemData(BaseModel): + scholar_id: str + cluster_id: str | None = None + fingerprint_sha256: str | None = None + title: str + year: int | None = None + citation_count: int = 0 + author_text: str | None = None + venue_text: str | None = None + pub_url: str | None = None + pdf_url: str | None = None + is_read: bool = False + + model_config = ConfigDict(extra="forbid") + + +class DataExportData(BaseModel): + schema_version: int + exported_at: str + scholars: list[ScholarExportItemData] + publications: list[PublicationExportItemData] + + model_config = ConfigDict(extra="forbid") + + +class DataExportEnvelope(BaseModel): + data: DataExportData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class DataImportRequest(BaseModel): + schema_version: int | None = None + scholars: list[ScholarExportItemData] = Field(default_factory=list) + publications: list[PublicationExportItemData] = Field(default_factory=list) + + model_config = ConfigDict(extra="forbid") + + +class DataImportResultData(BaseModel): + scholars_created: int + scholars_updated: int + publications_created: int + publications_updated: int + links_created: int + links_updated: int + skipped_records: int + + model_config = ConfigDict(extra="forbid") + + +class DataImportEnvelope(BaseModel): + data: DataImportResultData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") diff --git a/app/api/schemas/settings.py b/app/api/schemas/settings.py new file mode 100644 index 0000000..cc06ee8 --- /dev/null +++ b/app/api/schemas/settings.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from app.api.schemas.common import ApiMeta +from app.api.schemas.runs import ScrapeSafetyStateData + + +class SettingsPolicyData(BaseModel): + min_run_interval_minutes: int + min_request_delay_seconds: int + automation_allowed: bool + manual_run_allowed: bool + blocked_failure_threshold: int + network_failure_threshold: int + cooldown_blocked_seconds: int + cooldown_network_seconds: int + + model_config = ConfigDict(extra="forbid") + + +class SettingsData(BaseModel): + auto_run_enabled: bool + run_interval_minutes: int + request_delay_seconds: int + nav_visible_pages: list[str] + policy: SettingsPolicyData + safety_state: ScrapeSafetyStateData + + openalex_api_key: str | None = None + crossref_api_token: str | None = None + crossref_api_mailto: str | None = None + + model_config = ConfigDict(extra="forbid") + + +class SettingsEnvelope(BaseModel): + data: SettingsData + meta: ApiMeta + + model_config = ConfigDict(extra="forbid") + + +class SettingsUpdateRequest(BaseModel): + auto_run_enabled: bool + run_interval_minutes: int + request_delay_seconds: int + nav_visible_pages: list[str] | None = None + + openalex_api_key: str | None = None + crossref_api_token: str | None = None + crossref_api_mailto: str | None = None + + model_config = ConfigDict(extra="forbid") diff --git a/app/services/ingestion/application.py b/app/services/ingestion/application.py index abe23a3..f1a906e 100644 --- a/app/services/ingestion/application.py +++ b/app/services/ingestion/application.py @@ -1,68 +1,48 @@ from __future__ import annotations import asyncio -import hashlib import logging -import re -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from typing import Any -from sqlalchemy import or_, select, text +from sqlalchemy import select, text from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import ( CrawlRun, - Publication, RunStatus, RunTriggerType, ScholarProfile, - ScholarPublication, ) from app.logging_utils import structured_log -from app.services.arxiv.errors import ArxivRateLimitError from app.services.ingestion import queue as queue_service from app.services.ingestion import safety as run_safety_service -from app.services.ingestion.constants import ( - FAILED_STATES, - FAILURE_BUCKET_BLOCKED, - FAILURE_BUCKET_INGESTION, - FAILURE_BUCKET_LAYOUT, - FAILURE_BUCKET_NETWORK, - FAILURE_BUCKET_OTHER, - RESUMABLE_PARTIAL_REASON_PREFIXES, - RESUMABLE_PARTIAL_REASONS, - RUN_LOCK_NAMESPACE, -) -from app.services.ingestion.fingerprints import ( - _build_body_excerpt, -) +from app.services.ingestion.constants import RUN_LOCK_NAMESPACE +from app.services.ingestion.enrichment import EnrichmentRunner from app.services.ingestion.pagination import PaginationEngine -from app.services.ingestion.publication_upsert import upsert_profile_publications +from app.services.ingestion.run_completion import ( + _int_or_default, + complete_run_for_user, + run_execution_summary, +) +from app.services.ingestion.scholar_processing import run_scholar_iteration from app.services.ingestion.types import ( - PagedParseResult, RunAlertSummary, RunAlreadyInProgressError, RunBlockedBySafetyPolicyError, - RunExecutionSummary, RunFailureSummary, RunProgress, - ScholarProcessingOutcome, ) -from app.services.publication_identifiers import application as identifier_service -from app.services.runs.events import run_events -from app.services.scholar.parser import ( - ParsedProfilePage, - ParseState, - PublicationCandidate, -) -from app.services.scholar.source import FetchResult, ScholarSource +from app.services.scholar.source import ScholarSource from app.services.settings import application as user_settings_service from app.settings import settings logger = logging.getLogger(__name__) ACTIVE_RUN_INDEX_NAME = "uq_crawl_runs_user_active" +_background_tasks: set[asyncio.Task[Any]] = set() + def _is_active_run_integrity_error(exc: IntegrityError) -> bool: original_error = getattr(exc, "orig", None) @@ -78,35 +58,77 @@ def _is_active_run_integrity_error(exc: IntegrityError) -> bool: return getattr(diagnostics, "constraint_name", None) == ACTIVE_RUN_INDEX_NAME -def _int_or_default(value: Any, default: int = 0) -> int: - try: - return int(value) - except (TypeError, ValueError): - return default +def _resolve_paging_kwargs( + *, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int | None, + rate_limit_backoff_seconds: float | None, + max_pages_per_scholar: int, + page_size: int, +) -> dict[str, Any]: + return { + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "rate_limit_retries": rate_limit_retries + if rate_limit_retries is not None + else settings.ingestion_rate_limit_retries, + "rate_limit_backoff_seconds": rate_limit_backoff_seconds + if rate_limit_backoff_seconds is not None + else settings.ingestion_rate_limit_backoff_seconds, + "max_pages_per_scholar": max_pages_per_scholar, + "page_size": page_size, + } -def _classify_failure_bucket(*, state: str, state_reason: str) -> str: - reason = state_reason.strip().lower() - normalized_state = state.strip().lower() - - if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"): - return FAILURE_BUCKET_BLOCKED - if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"): - return FAILURE_BUCKET_NETWORK - if normalized_state == ParseState.LAYOUT_CHANGED.value: - return FAILURE_BUCKET_LAYOUT - if normalized_state == "ingestion_error": - return FAILURE_BUCKET_INGESTION - return FAILURE_BUCKET_OTHER +def _threshold_kwargs( + *, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, +) -> dict[str, Any]: + return { + "alert_blocked_failure_threshold": alert_blocked_failure_threshold, + "alert_network_failure_threshold": alert_network_failure_threshold, + "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, + } -_background_tasks: set[asyncio.Task[Any]] = set() +def _log_run_completed( + *, + run: CrawlRun, + user_id: int, + scholars: list[ScholarProfile], + progress: RunProgress, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, +) -> None: + structured_log( + logger, + "info", + "ingestion.run_completed", + user_id=user_id, + crawl_run_id=run.id, + status=run.status.value, + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + new_publication_count=run.new_pub_count, + blocked_failure_count=alert_summary.blocked_failure_count, + network_failure_count=alert_summary.network_failure_count, + retries_scheduled_count=failure_summary.retries_scheduled_count, + alert_flags=alert_summary.alert_flags, + ) class ScholarIngestionService: def __init__(self, *, source: ScholarSource) -> None: self._source = source self._pagination = PaginationEngine(source=source) + self._enrichment = EnrichmentRunner() @staticmethod def _effective_request_delay_seconds(value: int) -> int: @@ -122,15 +144,9 @@ class ScholarIngestionService: user_id: int, trigger_type: RunTriggerType, ): - user_settings = await user_settings_service.get_or_create_settings( - db_session, - user_id=user_id, - ) + user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) await self._enforce_safety_gate( - db_session, - user_settings=user_settings, - user_id=user_id, - trigger_type=trigger_type, + db_session, user_settings=user_settings, user_id=user_id, trigger_type=trigger_type ) return user_settings @@ -174,10 +190,7 @@ class ScholarIngestionService: trigger_type: RunTriggerType, now_utc: datetime, ) -> None: - safety_state = run_safety_service.register_cooldown_blocked_start( - user_settings, - now_utc=now_utc, - ) + safety_state = run_safety_service.register_cooldown_blocked_start(user_settings, now_utc=now_utc) await db_session.commit() structured_log( logger, @@ -196,18 +209,6 @@ class ScholarIngestionService: safety_state=safety_state, ) - @staticmethod - def _normalize_run_targets( - *, - scholar_profile_ids: set[int] | None, - start_cstart_by_scholar_id: dict[int, int] | None, - ) -> tuple[set[int] | None, dict[int, int]]: - filtered_scholar_ids = ( - {int(value) for value in scholar_profile_ids} if scholar_profile_ids is not None else None - ) - start_cstart_map = {int(key): max(0, int(value)) for key, value in (start_cstart_by_scholar_id or {}).items()} - return filtered_scholar_ids, start_cstart_map - async def _load_target_scholars( self, db_session: AsyncSession, @@ -224,844 +225,12 @@ class ScholarIngestionService: scholars_stmt = scholars_stmt.where(ScholarProfile.id.in_(filtered_scholar_ids)) scholars_result = await db_session.execute(scholars_stmt) scholars = list(scholars_result.scalars().all()) - await self._clear_missing_filtered_jobs( - db_session, - user_id=user_id, - filtered_scholar_ids=filtered_scholar_ids, - scholars=scholars, - ) + if filtered_scholar_ids is not None: + found_ids = {int(s.id) for s in scholars} + for sid in filtered_scholar_ids - found_ids: + await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=sid) return scholars - async def _clear_missing_filtered_jobs( - self, - db_session: AsyncSession, - *, - user_id: int, - filtered_scholar_ids: set[int] | None, - scholars: list[ScholarProfile], - ) -> None: - if filtered_scholar_ids is None: - return - found_ids = {int(scholar.id) for scholar in scholars} - missing_ids = filtered_scholar_ids - found_ids - for scholar_profile_id in missing_ids: - await queue_service.clear_job_for_scholar( - db_session, - user_id=user_id, - scholar_profile_id=scholar_profile_id, - ) - - @staticmethod - def _create_running_run( - *, - user_id: int, - trigger_type: RunTriggerType, - scholar_count: int, - idempotency_key: str | None, - ) -> CrawlRun: - return CrawlRun( - user_id=user_id, - trigger_type=trigger_type, - status=RunStatus.RUNNING, - scholar_count=scholar_count, - new_pub_count=0, - idempotency_key=idempotency_key, - error_log={}, - ) - - @staticmethod - async def _wait_between_scholars(*, index: int, request_delay_seconds: int) -> None: - if index <= 0 or request_delay_seconds <= 0: - return - await asyncio.sleep(float(request_delay_seconds)) - - @staticmethod - def _assert_valid_paged_parse_result( - *, - scholar_id: str, - paged_parse_result: PagedParseResult, - ) -> None: - parsed_page = paged_parse_result.parsed_page - if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any( - code.startswith("layout_") for code in parsed_page.warnings - ): - raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") - for publication in paged_parse_result.publications: - if not publication.title.strip(): - raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") - if publication.citation_count is not None and int(publication.citation_count) < 0: - raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") - - @staticmethod - def _apply_first_page_profile_metadata( - *, - scholar: ScholarProfile, - paged_parse_result: PagedParseResult, - run_dt: datetime, - ) -> None: - first_page = paged_parse_result.first_page_parsed_page - if first_page.profile_name and not (scholar.display_name or "").strip(): - scholar.display_name = first_page.profile_name - if first_page.profile_image_url: - scholar.profile_image_url = first_page.profile_image_url - scholar.last_initial_page_checked_at = run_dt - - @staticmethod - def _build_result_entry( - *, - scholar: ScholarProfile, - start_cstart: int, - paged_parse_result: PagedParseResult, - ) -> dict[str, Any]: - parsed_page = paged_parse_result.parsed_page - return { - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": parsed_page.state.value, - "state_reason": parsed_page.state_reason, - "outcome": "failed", - "attempt_count": len(paged_parse_result.attempt_log), - "publication_count": len(paged_parse_result.publications), - "start_cstart": start_cstart, - "articles_range": parsed_page.articles_range, - "warnings": parsed_page.warnings, - "has_show_more_button": parsed_page.has_show_more_button, - "pages_fetched": paged_parse_result.pages_fetched, - "pages_attempted": paged_parse_result.pages_attempted, - "has_more_remaining": paged_parse_result.has_more_remaining, - "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, - "continuation_cstart": paged_parse_result.continuation_cstart, - "skipped_no_change": paged_parse_result.skipped_no_change, - "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - } - - def _skipped_no_change_outcome( - self, - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - first_page = paged_parse_result.first_page_parsed_page - scholar.last_run_status = RunStatus.SUCCESS - scholar.last_run_dt = run_dt - result_entry["state"] = first_page.state.value - result_entry["state_reason"] = "no_change_initial_page_signature" - result_entry["outcome"] = "success" - result_entry["publication_count"] = 0 - result_entry["warnings"] = first_page.warnings - result_entry["debug"] = { - "state_reason": "no_change_initial_page_signature", - "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, - "attempt_log": paged_parse_result.attempt_log, - "page_logs": paged_parse_result.page_logs, - } - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=0, - discovered_publication_count=0, - ) - - async def _upsert_publications_outcome( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - parsed_page = paged_parse_result.parsed_page - publications = paged_parse_result.publications - had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} - has_partial_set = len(publications) > 0 and had_page_failure - if (not had_page_failure) or has_partial_set: - return await self._upsert_success_or_exception( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_set, - ) - return self._parse_failure_outcome( - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - - async def _upsert_success_or_exception( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, - ) -> ScholarProcessingOutcome: - try: - return await self._upsert_success( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - has_partial_publication_set=has_partial_publication_set, - ) - except Exception as exc: - return self._upsert_exception_outcome( - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - exc=exc, - ) - - async def _upsert_success( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - has_partial_publication_set: bool, - ) -> ScholarProcessingOutcome: - # We no longer aggregate and upsert the publications here. - # Eager DB insertions have already saved and committed them inside `fetch_and_parse_all_pages` and `_paginate_loop`. - discovered_count = paged_parse_result.discovered_publication_count - is_partial = ( - paged_parse_result.has_more_remaining - or paged_parse_result.pagination_truncated_reason is not None - or has_partial_publication_set - ) - scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS - scholar.last_run_dt = run_dt - if not is_partial and paged_parse_result.first_page_fingerprint_sha256: - scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 - result_entry["outcome"] = "partial" if is_partial else "success" - if is_partial: - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - return ScholarProcessingOutcome( - result_entry=result_entry, - succeeded_count_delta=1, - failed_count_delta=0, - partial_count_delta=1 if is_partial else 0, - discovered_publication_count=discovered_count, - ) - - def _upsert_exception_outcome( - self, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - exc: Exception, - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["state"] = "ingestion_error" - result_entry["state_reason"] = "publication_upsert_exception" - result_entry["outcome"] = "failed" - result_entry["error"] = str(exc) - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - exception=exc, - ) - logger.exception( - "ingestion.scholar_failed", - extra={ - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - def _parse_failure_outcome( - self, - *, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = run_dt - result_entry["debug"] = self._build_failure_debug_context( - fetch_result=paged_parse_result.fetch_result, - parsed_page=paged_parse_result.parsed_page, - attempt_log=paged_parse_result.attempt_log, - page_logs=paged_parse_result.page_logs, - ) - structured_log( - logger, - "warning", - "ingestion.scholar_parse_failed", - scholar_profile_id=scholar.id, - scholar_id=scholar.scholar_id, - state=paged_parse_result.parsed_page.state.value, - state_reason=paged_parse_result.parsed_page.state_reason, - status_code=paged_parse_result.fetch_result.status_code, - ) - return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) - - async def _sync_continuation_queue( - self, - db_session: AsyncSession, - *, - user_id: int, - scholar: ScholarProfile, - run: CrawlRun, - start_cstart: int, - result_entry: dict[str, Any], - paged_parse_result: PagedParseResult, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> None: - queue_reason, queue_cstart = self._resolve_continuation_queue_target( - outcome=str(result_entry.get("outcome", "")), - state=str(result_entry.get("state", "")), - pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, - continuation_cstart=paged_parse_result.continuation_cstart, - fallback_cstart=start_cstart, - ) - if auto_queue_continuations and queue_reason is not None: - await queue_service.upsert_job( - db_session, - user_id=user_id, - scholar_profile_id=scholar.id, - resume_cstart=queue_cstart, - reason=queue_reason, - run_id=run.id, - delay_seconds=queue_delay_seconds, - ) - result_entry["continuation_enqueued"] = True - result_entry["continuation_reason"] = queue_reason - result_entry["continuation_cstart"] = queue_cstart - return - if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id): - result_entry["continuation_cleared"] = True - - async def _process_scholar( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - start_cstart: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> ScholarProcessingOutcome: - try: - return await self._process_scholar_inner( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - start_cstart=start_cstart, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - except Exception as exc: - return self._unexpected_scholar_exception_outcome( - run=run, - scholar=scholar, - start_cstart=start_cstart, - exc=exc, - ) - - async def _process_scholar_inner( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - start_cstart: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> ScholarProcessingOutcome: - run_dt, paged_parse_result, result_entry = await self._fetch_and_prepare_scholar_result( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - start_cstart=start_cstart, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - ) - outcome = await self._resolve_scholar_outcome( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - await self._sync_continuation_queue( - db_session, - user_id=user_id, - scholar=scholar, - run=run, - start_cstart=start_cstart, - result_entry=outcome.result_entry, - paged_parse_result=paged_parse_result, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - return outcome - - async def _fetch_and_prepare_scholar_result( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - user_id: int, - start_cstart: int, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - ) -> tuple[datetime, PagedParseResult, dict[str, Any]]: - run_dt = datetime.now(UTC) - paged_parse_result = await self._pagination.fetch_and_parse_all_pages( - scholar=scholar, - run=run, - db_session=db_session, - start_cstart=start_cstart, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages=max_pages_per_scholar, - page_size=page_size, - previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, - upsert_publications_fn=upsert_profile_publications, - ) - self._assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result) - self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt) - parsed_page = paged_parse_result.parsed_page - structured_log( - logger, - "info", - "ingestion.scholar_parsed", - user_id=user_id, - crawl_run_id=run.id, - scholar_profile_id=scholar.id, - scholar_id=scholar.scholar_id, - state=parsed_page.state.value, - publication_count=len(paged_parse_result.publications), - has_show_more_button=parsed_page.has_show_more_button, - pages_fetched=paged_parse_result.pages_fetched, - pages_attempted=paged_parse_result.pages_attempted, - has_more_remaining=paged_parse_result.has_more_remaining, - pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, - warning_count=len(parsed_page.warnings), - skipped_no_change=paged_parse_result.skipped_no_change, - ) - result_entry = self._build_result_entry( - scholar=scholar, - start_cstart=start_cstart, - paged_parse_result=paged_parse_result, - ) - return run_dt, paged_parse_result, result_entry - - async def _resolve_scholar_outcome( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholar: ScholarProfile, - run_dt: datetime, - paged_parse_result: PagedParseResult, - result_entry: dict[str, Any], - ) -> ScholarProcessingOutcome: - if paged_parse_result.skipped_no_change: - return self._skipped_no_change_outcome( - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - return await self._upsert_publications_outcome( - db_session, - run=run, - scholar=scholar, - run_dt=run_dt, - paged_parse_result=paged_parse_result, - result_entry=result_entry, - ) - - @staticmethod - def _result_counters(result_entry: dict[str, Any]) -> tuple[int, int, int]: - outcome = str(result_entry.get("outcome", "")).strip().lower() - if outcome == "success": - return 1, 0, 0 - if outcome == "partial": - return 1, 0, 1 - if outcome == "failed": - return 0, 1, 0 - raise RuntimeError(f"Unexpected scholar outcome label: {outcome!r}") - - @staticmethod - def _find_scholar_result_index( - *, - scholar_results: list[dict[str, Any]], - scholar_profile_id: int, - ) -> int | None: - for index, result_entry in enumerate(scholar_results): - current_scholar_id = _int_or_default(result_entry.get("scholar_profile_id"), 0) - if current_scholar_id == scholar_profile_id: - return index - return None - - @staticmethod - def _adjust_progress_counts( - *, - progress: RunProgress, - succeeded_delta: int, - failed_delta: int, - partial_delta: int, - ) -> None: - progress.succeeded_count += succeeded_delta - progress.failed_count += failed_delta - progress.partial_count += partial_delta - if progress.succeeded_count < 0 or progress.failed_count < 0 or progress.partial_count < 0: - raise RuntimeError("RunProgress counters entered invalid negative state.") - - @staticmethod - def _apply_outcome_to_progress( - *, - progress: RunProgress, - outcome: ScholarProcessingOutcome, - ) -> None: - scholar_profile_id = _int_or_default(outcome.result_entry.get("scholar_profile_id"), 0) - if scholar_profile_id <= 0: - raise RuntimeError("Scholar outcome missing valid scholar_profile_id.") - prior_index = ScholarIngestionService._find_scholar_result_index( - scholar_results=progress.scholar_results, - scholar_profile_id=scholar_profile_id, - ) - next_succeeded, next_failed, next_partial = ScholarIngestionService._result_counters(outcome.result_entry) - if prior_index is None: - progress.scholar_results.append(outcome.result_entry) - ScholarIngestionService._adjust_progress_counts( - progress=progress, - succeeded_delta=next_succeeded, - failed_delta=next_failed, - partial_delta=next_partial, - ) - return - previous_entry = progress.scholar_results[prior_index] - prev_succeeded, prev_failed, prev_partial = ScholarIngestionService._result_counters(previous_entry) - progress.scholar_results[prior_index] = outcome.result_entry - ScholarIngestionService._adjust_progress_counts( - progress=progress, - succeeded_delta=next_succeeded - prev_succeeded, - failed_delta=next_failed - prev_failed, - partial_delta=next_partial - prev_partial, - ) - - def _unexpected_scholar_exception_outcome( - self, - *, - run: CrawlRun, - scholar: ScholarProfile, - start_cstart: int, - exc: Exception, - ) -> ScholarProcessingOutcome: - scholar.last_run_status = RunStatus.FAILED - scholar.last_run_dt = datetime.now(UTC) - logger.exception( - "ingestion.scholar_unexpected_failure", - extra={ - "crawl_run_id": run.id, - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - }, - ) - return ScholarProcessingOutcome( - result_entry={ - "scholar_profile_id": scholar.id, - "scholar_id": scholar.scholar_id, - "state": "ingestion_error", - "state_reason": "scholar_processing_exception", - "outcome": "failed", - "attempt_count": 0, - "publication_count": 0, - "start_cstart": start_cstart, - "warnings": [], - "error": str(exc), - "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, - }, - succeeded_count_delta=0, - failed_count_delta=1, - partial_count_delta=0, - discovered_publication_count=0, - ) - - @staticmethod - def _summarize_failures( - *, - scholar_results: list[dict[str, Any]], - ) -> RunFailureSummary: - failed_state_counts: dict[str, int] = {} - failed_reason_counts: dict[str, int] = {} - scrape_failure_counts: dict[str, int] = {} - retries_scheduled_count = 0 - scholars_with_retries_count = 0 - retry_exhausted_count = 0 - for entry in scholar_results: - retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1) - if retries_for_entry > 0: - retries_scheduled_count += retries_for_entry - scholars_with_retries_count += 1 - if str(entry.get("outcome", "")) != "failed": - continue - state = str(entry.get("state", "")).strip() - if state not in FAILED_STATES: - continue - failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 - reason = str(entry.get("state_reason", "")).strip() - if reason: - failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 - bucket = _classify_failure_bucket(state=state, state_reason=reason) - scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1 - if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0: - retry_exhausted_count += 1 - return RunFailureSummary( - failed_state_counts=failed_state_counts, - failed_reason_counts=failed_reason_counts, - scrape_failure_counts=scrape_failure_counts, - retries_scheduled_count=retries_scheduled_count, - scholars_with_retries_count=scholars_with_retries_count, - retry_exhausted_count=retry_exhausted_count, - ) - - @staticmethod - def _build_alert_summary( - *, - failure_summary: RunFailureSummary, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> RunAlertSummary: - blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) - network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) - blocked_threshold = max(1, int(alert_blocked_failure_threshold)) - network_threshold = max(1, int(alert_network_failure_threshold)) - retry_threshold = max(1, int(alert_retry_scheduled_threshold)) - alert_flags = { - "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold, - "network_failure_threshold_exceeded": network_failure_count >= network_threshold, - "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold, - } - return RunAlertSummary( - blocked_failure_count=blocked_failure_count, - network_failure_count=network_failure_count, - blocked_failure_threshold=blocked_threshold, - network_failure_threshold=network_threshold, - retry_scheduled_threshold=retry_threshold, - alert_flags=alert_flags, - ) - - @staticmethod - def _apply_safety_outcome( - *, - user_settings, - run: CrawlRun, - user_id: int, - alert_summary: RunAlertSummary, - ) -> None: - pre_apply_state = run_safety_service.get_safety_event_context( - user_settings, - now_utc=datetime.now(UTC), - ) - safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome( - user_settings, - run_id=int(run.id), - blocked_failure_count=alert_summary.blocked_failure_count, - network_failure_count=alert_summary.network_failure_count, - blocked_failure_threshold=alert_summary.blocked_failure_threshold, - network_failure_threshold=alert_summary.network_failure_threshold, - blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, - network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, - now_utc=datetime.now(UTC), - ) - if cooldown_trigger_reason is not None: - structured_log( - logger, - "warning", - "ingestion.safety_cooldown_entered", - user_id=user_id, - crawl_run_id=int(run.id), - reason=cooldown_trigger_reason, - blocked_failure_count=alert_summary.blocked_failure_count, - network_failure_count=alert_summary.network_failure_count, - blocked_failure_threshold=alert_summary.blocked_failure_threshold, - network_failure_threshold=alert_summary.network_failure_threshold, - cooldown_until=safety_state.get("cooldown_until"), - cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"), - safety_counters=safety_state.get("counters", {}), - ) - elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"): - structured_log( - logger, - "info", - "ingestion.cooldown_cleared", - user_id=user_id, - crawl_run_id=int(run.id), - reason=pre_apply_state.get("cooldown_reason"), - cooldown_until=pre_apply_state.get("cooldown_until"), - ) - - def _finalize_run_record( - self, - *, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - failure_summary: RunFailureSummary, - alert_summary: RunAlertSummary, - idempotency_key: str | None, - ) -> None: - run.end_dt = datetime.now(UTC) - if run.status != RunStatus.CANCELED: - run.status = self._resolve_run_status( - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - ) - run.error_log = { - "scholar_results": progress.scholar_results, - "summary": { - "succeeded_count": progress.succeeded_count, - "failed_count": progress.failed_count, - "partial_count": progress.partial_count, - "failed_state_counts": failure_summary.failed_state_counts, - "failed_reason_counts": failure_summary.failed_reason_counts, - "scrape_failure_counts": failure_summary.scrape_failure_counts, - "retry_counts": { - "retries_scheduled_count": failure_summary.retries_scheduled_count, - "scholars_with_retries_count": failure_summary.scholars_with_retries_count, - "retry_exhausted_count": failure_summary.retry_exhausted_count, - }, - "alert_thresholds": { - "blocked_failure_threshold": alert_summary.blocked_failure_threshold, - "network_failure_threshold": alert_summary.network_failure_threshold, - "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold, - }, - "alert_flags": alert_summary.alert_flags, - }, - "meta": {"idempotency_key": idempotency_key} if idempotency_key else {}, - } - - @staticmethod - def _paging_kwargs( - *, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - ) -> dict[str, Any]: - return { - "request_delay_seconds": request_delay_seconds, - "network_error_retries": network_error_retries, - "retry_backoff_seconds": retry_backoff_seconds, - "rate_limit_retries": rate_limit_retries, - "rate_limit_backoff_seconds": rate_limit_backoff_seconds, - "max_pages_per_scholar": max_pages_per_scholar, - "page_size": page_size, - } - - @staticmethod - def _threshold_kwargs( - *, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> dict[str, Any]: - return { - "alert_blocked_failure_threshold": alert_blocked_failure_threshold, - "alert_network_failure_threshold": alert_network_failure_threshold, - "alert_retry_scheduled_threshold": alert_retry_scheduled_threshold, - } - - @staticmethod - def _run_execution_summary( - *, - run: CrawlRun, - scholars: list[ScholarProfile], - progress: RunProgress, - ) -> RunExecutionSummary: - return RunExecutionSummary( - crawl_run_id=run.id, - status=run.status, - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - new_publication_count=run.new_pub_count, - ) - async def _initialize_run_for_user( self, db_session: AsyncSession, @@ -1070,17 +239,9 @@ class ScholarIngestionService: trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, + paging_kwargs: dict[str, Any], + threshold_kwargs: dict[str, Any], idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, ) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]: user_settings = await self._load_user_settings_for_run( db_session, @@ -1089,34 +250,26 @@ class ScholarIngestionService: ) if not await self._try_acquire_user_lock(db_session, user_id=user_id): raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") - filtered_scholar_ids, start_cstart_map = self._normalize_run_targets( - scholar_profile_ids=scholar_profile_ids, - start_cstart_by_scholar_id=start_cstart_by_scholar_id, - ) + filtered_scholar_ids = {int(v) for v in scholar_profile_ids} if scholar_profile_ids is not None else None + start_cstart_map = {int(k): max(0, int(v)) for k, v in (start_cstart_by_scholar_id or {}).items()} scholars = await self._load_target_scholars( db_session, user_id=user_id, filtered_scholar_ids=filtered_scholar_ids, ) - run = await self._start_run_record_for_targets( + run = await self._start_run_record( db_session, user_id=user_id, trigger_type=trigger_type, scholars=scholars, filtered=filtered_scholar_ids is not None, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, idempotency_key=idempotency_key, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + paging_kwargs=paging_kwargs, + threshold_kwargs=threshold_kwargs, ) return user_settings, run, scholars, start_cstart_map - async def _start_run_record_for_targets( + async def _start_run_record( self, db_session: AsyncSession, *, @@ -1124,15 +277,9 @@ class ScholarIngestionService: trigger_type: RunTriggerType, scholars: list[ScholarProfile], filtered: bool, - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, + paging_kwargs: dict[str, Any], + threshold_kwargs: dict[str, Any], ) -> CrawlRun: structured_log( logger, @@ -1142,21 +289,18 @@ class ScholarIngestionService: trigger_type=trigger_type.value, scholar_count=len(scholars), is_filtered_run=filtered, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, idempotency_key=idempotency_key, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + **paging_kwargs, + **threshold_kwargs, ) - run = self._create_running_run( + run = CrawlRun( user_id=user_id, trigger_type=trigger_type, + status=RunStatus.RUNNING, scholar_count=len(scholars), + new_pub_count=0, idempotency_key=idempotency_key, + error_log={}, ) db_session.add(run) try: @@ -1168,165 +312,6 @@ class ScholarIngestionService: raise return run - async def _run_scholar_iteration( - self, - db_session: AsyncSession, - *, - run: CrawlRun, - scholars: list[ScholarProfile], - user_id: int, - start_cstart_map: dict[int, int], - request_delay_seconds: int, - network_error_retries: int, - retry_backoff_seconds: float, - rate_limit_retries: int, - rate_limit_backoff_seconds: float, - max_pages_per_scholar: int, - page_size: int, - auto_queue_continuations: bool, - queue_delay_seconds: int, - ) -> RunProgress: - progress = RunProgress() - - # ── Pass 1: first page of every scholar (breadth-first) ────────── - # This ensures all scholars have visible publications as quickly as - # possible, rather than fully paginating one scholar before the next. - first_pass_cstarts: dict[int, int] = {} - for index, scholar in enumerate(scholars): - await db_session.refresh(run) - if run.status == RunStatus.CANCELED: - structured_log( - logger, - "info", - "ingestion.run_canceled", - run_id=run.id, - user_id=user_id, - ) - return progress - await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) - start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) - outcome = await self._process_scholar( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages_per_scholar=1, - page_size=page_size, - start_cstart=start_cstart, - auto_queue_continuations=False, - queue_delay_seconds=queue_delay_seconds, - ) - self._apply_outcome_to_progress(progress=progress, outcome=outcome) - # Track where to resume from for the deep pass - resume_cstart = outcome.result_entry.get("continuation_cstart") - if resume_cstart is not None and int(resume_cstart) > start_cstart: - first_pass_cstarts[int(scholar.id)] = int(resume_cstart) - - # ── Pass 2: remaining pages for each scholar (depth) ───────────── - remaining_max = max(max_pages_per_scholar - 1, 0) - if remaining_max <= 0: - return progress - - for index, scholar in enumerate(scholars): - resume_cstart = first_pass_cstarts.get(int(scholar.id)) - if resume_cstart is None: - continue # first page failed, had no continuation, or was skipped - await db_session.refresh(run) - if run.status == RunStatus.CANCELED: - structured_log( - logger, - "info", - "ingestion.run_canceled", - run_id=run.id, - user_id=user_id, - ) - break - await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds) - outcome = await self._process_scholar( - db_session, - run=run, - scholar=scholar, - user_id=user_id, - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds, - max_pages_per_scholar=remaining_max, - page_size=page_size, - start_cstart=resume_cstart, - auto_queue_continuations=auto_queue_continuations, - queue_delay_seconds=queue_delay_seconds, - ) - self._apply_outcome_to_progress(progress=progress, outcome=outcome) - return progress - - def _complete_run_for_user( - self, - *, - user_settings: Any, - run: CrawlRun, - scholars: list[ScholarProfile], - user_id: int, - progress: RunProgress, - idempotency_key: str | None, - alert_blocked_failure_threshold: int, - alert_network_failure_threshold: int, - alert_retry_scheduled_threshold: int, - ) -> tuple[RunFailureSummary, RunAlertSummary]: - failure_summary = self._summarize_failures(scholar_results=progress.scholar_results) - alert_summary = self._build_alert_summary( - failure_summary=failure_summary, - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]: - structured_log( - logger, - "warning", - "ingestion.alert_blocked_failure_threshold_exceeded", - user_id=user_id, - crawl_run_id=int(run.id), - blocked_failure_count=alert_summary.blocked_failure_count, - threshold=alert_summary.blocked_failure_threshold, - ) - if alert_summary.alert_flags["network_failure_threshold_exceeded"]: - structured_log( - logger, - "warning", - "ingestion.alert_network_failure_threshold_exceeded", - user_id=user_id, - crawl_run_id=int(run.id), - network_failure_count=alert_summary.network_failure_count, - threshold=alert_summary.network_failure_threshold, - ) - if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]: - structured_log( - logger, - "warning", - "ingestion.alert_retry_scheduled_threshold_exceeded", - user_id=user_id, - crawl_run_id=int(run.id), - retries_scheduled_count=failure_summary.retries_scheduled_count, - threshold=alert_summary.retry_scheduled_threshold, - ) - self._apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary) - self._finalize_run_record( - run=run, - scholars=scholars, - progress=progress, - failure_summary=failure_summary, - alert_summary=alert_summary, - idempotency_key=idempotency_key, - ) - return failure_summary, alert_summary - async def initialize_run( self, db_session: AsyncSession, @@ -1360,26 +345,20 @@ class ScholarIngestionService: settings.ingestion_min_request_delay_seconds ), ) - - paging_kwargs = self._paging_kwargs( + paging = _resolve_paging_kwargs( request_delay_seconds=effective_delay, network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries - if rate_limit_retries is not None - else settings.ingestion_rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds - if rate_limit_backoff_seconds is not None - else settings.ingestion_rate_limit_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size, ) - threshold_kwargs = self._threshold_kwargs( + thresholds = _threshold_kwargs( alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, ) - _, run, scholars, start_cstart_map = await self._initialize_run_for_user( db_session, user_id=user_id, @@ -1387,8 +366,8 @@ class ScholarIngestionService: scholar_profile_ids=scholar_profile_ids, start_cstart_by_scholar_id=start_cstart_by_scholar_id, idempotency_key=idempotency_key, - **paging_kwargs, - **threshold_kwargs, + paging_kwargs=paging, + threshold_kwargs=thresholds, ) return run, scholars, start_cstart_map @@ -1414,78 +393,57 @@ class ScholarIngestionService: alert_retry_scheduled_threshold: int = 3, idempotency_key: str | None = None, ) -> None: + paging = _resolve_paging_kwargs( + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + thresholds = _threshold_kwargs( + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) async with session_factory() as db_session: try: - # Re-fetch everything to ensure attachment to the new session run, user_settings, attached_scholars = await self._prepare_execute_run( db_session, run_id=run_id, user_id=user_id, scholars=scholars ) - - paging_kwargs = self._paging_kwargs( - request_delay_seconds=request_delay_seconds, - network_error_retries=network_error_retries, - retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries - if rate_limit_retries is not None - else settings.ingestion_rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds - if rate_limit_backoff_seconds is not None - else settings.ingestion_rate_limit_backoff_seconds, - max_pages_per_scholar=max_pages_per_scholar, - page_size=page_size, - ) - threshold_kwargs = self._threshold_kwargs( - alert_blocked_failure_threshold=alert_blocked_failure_threshold, - alert_network_failure_threshold=alert_network_failure_threshold, - alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, - ) - - progress = await self._run_scholar_iteration( + progress = await run_scholar_iteration( db_session, + pagination=self._pagination, run=run, scholars=attached_scholars, user_id=user_id, start_cstart_map=start_cstart_map, auto_queue_continuations=auto_queue_continuations, queue_delay_seconds=queue_delay_seconds, - **paging_kwargs, + **paging, ) - - failure_summary, alert_summary = self._complete_run_for_user( + failure_summary, alert_summary = complete_run_for_user( user_settings=user_settings, run=run, scholars=attached_scholars, user_id=user_id, progress=progress, idempotency_key=idempotency_key, - **threshold_kwargs, + **thresholds, ) - - # Capture the final status that _finalize_run_record computed, - # then set RESOLVING so the UI shows enrichment is in progress. intended_final_status = run.status if intended_final_status not in (RunStatus.CANCELED,): run.status = RunStatus.RESOLVING await db_session.commit() - structured_log( - logger, - "info", - "ingestion.run_completed", + _log_run_completed( + run=run, user_id=user_id, - crawl_run_id=run.id, - status=run.status.value, - scholar_count=len(attached_scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - new_publication_count=run.new_pub_count, - blocked_failure_count=alert_summary.blocked_failure_count, - network_failure_count=alert_summary.network_failure_count, - retries_scheduled_count=failure_summary.retries_scheduled_count, - alert_flags=alert_summary.alert_flags, + scholars=attached_scholars, + progress=progress, + failure_summary=failure_summary, + alert_summary=alert_summary, ) - - # Fire-and-forget enrichment in a separate background task if intended_final_status not in (RunStatus.CANCELED,): task = asyncio.create_task( self._background_enrich( @@ -1513,7 +471,6 @@ class ScholarIngestionService: run_result = await db_session.execute(select(CrawlRun).where(CrawlRun.id == run_id)) run = run_result.scalar_one() user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) - scholar_ids = [s.id for s in scholars] scholars_result = await db_session.execute( select(ScholarProfile) @@ -1539,10 +496,9 @@ class ScholarIngestionService: intended_final_status: RunStatus, openalex_api_key: str | None = None, ) -> None: - """Run enrichment independently — failures don't affect run status.""" try: async with session_factory() as session: - await self._enrich_pending_publications( + await self._enrichment.enrich_pending_publications( session, run_id=run_id, openalex_api_key=openalex_api_key, @@ -1556,11 +512,7 @@ class ScholarIngestionService: extra={"run_id": run_id, "final_status": str(intended_final_status)}, ) except Exception: - logger.exception( - "ingestion.background_enrichment_failed", - extra={"run_id": run_id}, - ) - # Still transition out of RESOLVING so the run doesn't stay stuck. + logger.exception("ingestion.background_enrichment_failed", extra={"run_id": run_id}) try: async with session_factory() as fallback_session: run = await fallback_session.get(CrawlRun, run_id) @@ -1568,10 +520,7 @@ class ScholarIngestionService: run.status = intended_final_status await fallback_session.commit() except Exception: - logger.exception( - "ingestion.background_enrichment_fallback_failed", - extra={"run_id": run_id}, - ) + logger.exception("ingestion.background_enrichment_fallback_failed", extra={"run_id": run_id}) async def run_for_user( self, @@ -1594,8 +543,7 @@ class ScholarIngestionService: alert_blocked_failure_threshold: int = 1, alert_network_failure_threshold: int = 2, alert_retry_scheduled_threshold: int = 3, - ) -> RunExecutionSummary: - # Legacy/Synchronous trigger (used by tests or older flows) + ): run, scholars, start_cstart_map = await self.initialize_run( db_session, user_id=user_id, @@ -1614,459 +562,65 @@ class ScholarIngestionService: alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, ) - - paging_kwargs = self._paging_kwargs( + paging = _resolve_paging_kwargs( request_delay_seconds=self._effective_request_delay_seconds(request_delay_seconds), network_error_retries=network_error_retries, retry_backoff_seconds=retry_backoff_seconds, - rate_limit_retries=rate_limit_retries - if rate_limit_retries is not None - else settings.ingestion_rate_limit_retries, - rate_limit_backoff_seconds=rate_limit_backoff_seconds - if rate_limit_backoff_seconds is not None - else settings.ingestion_rate_limit_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, max_pages_per_scholar=max_pages_per_scholar, page_size=page_size, ) - - threshold_kwargs = self._threshold_kwargs( + thresholds = _threshold_kwargs( alert_blocked_failure_threshold=alert_blocked_failure_threshold, alert_network_failure_threshold=alert_network_failure_threshold, alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, ) - - progress = await self._run_scholar_iteration( + progress = await run_scholar_iteration( db_session, + pagination=self._pagination, run=run, scholars=scholars, user_id=user_id, start_cstart_map=start_cstart_map, auto_queue_continuations=auto_queue_continuations, queue_delay_seconds=queue_delay_seconds, - **paging_kwargs, + **paging, ) - user_settings = await user_settings_service.get_or_create_settings(db_session, user_id=user_id) - failure_summary, alert_summary = self._complete_run_for_user( + failure_summary, alert_summary = complete_run_for_user( user_settings=user_settings, run=run, scholars=scholars, user_id=user_id, progress=progress, idempotency_key=idempotency_key, - **threshold_kwargs, + **thresholds, ) - - # Set RESOLVING during enrichment (synchronous path). intended_final_status = run.status if intended_final_status not in (RunStatus.CANCELED,): run.status = RunStatus.RESOLVING await db_session.commit() - try: - await self._enrich_pending_publications( + await self._enrichment.enrich_pending_publications( db_session, run_id=run.id, openalex_api_key=getattr(user_settings, "openalex_api_key", None), ) except Exception: logger.exception("ingestion.enrichment_failed", extra={"run_id": run.id}) - - # Finalize to the intended status unless the run was canceled during resolving. if run.status == RunStatus.RESOLVING: run.status = intended_final_status await db_session.commit() - - structured_log( - logger, - "info", - "ingestion.run_completed", + _log_run_completed( + run=run, user_id=user_id, - crawl_run_id=run.id, - status=run.status.value, - scholar_count=len(scholars), - succeeded_count=progress.succeeded_count, - failed_count=progress.failed_count, - partial_count=progress.partial_count, - new_publication_count=run.new_pub_count, - blocked_failure_count=alert_summary.blocked_failure_count, - network_failure_count=alert_summary.network_failure_count, - retries_scheduled_count=failure_summary.retries_scheduled_count, - alert_flags=alert_summary.alert_flags, + scholars=scholars, + progress=progress, + failure_summary=failure_summary, + alert_summary=alert_summary, ) - return self._run_execution_summary(run=run, scholars=scholars, progress=progress) - - async def _run_is_canceled( - self, - db_session: AsyncSession, - *, - run_id: int, - ) -> bool: - check_result = await db_session.execute(select(CrawlRun.status).where(CrawlRun.id == run_id)) - status = check_result.scalar_one_or_none() - if status is None: - raise RuntimeError(f"Missing crawl_run for run_id={run_id}.") - return status == RunStatus.CANCELED - - async def _enrich_pending_publications( - self, - db_session: AsyncSession, - *, - run_id: int, - openalex_api_key: str | None = None, - ) -> None: - """Enrich unenriched publications with OpenAlex data. - - Stops immediately on budget exhaustion (429 with $0 remaining). - Sleeps 60s and continues on transient rate limits. - """ - from app.services.openalex.client import ( - OpenAlexBudgetExhaustedError, - OpenAlexClient, - OpenAlexRateLimitError, - ) - from app.services.openalex.matching import find_best_match - - run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id)) - user_id = run_result.scalar_one() - - now = datetime.now(UTC) - cooldown_threshold = now - timedelta(days=7) - - stmt = ( - select(Publication) - .join(ScholarPublication) - .join(ScholarProfile, ScholarPublication.scholar_profile_id == ScholarProfile.id) - .where( - ScholarProfile.user_id == user_id, - Publication.openalex_enriched.is_(False), - or_( - Publication.openalex_last_attempt_at.is_(None), - Publication.openalex_last_attempt_at < cooldown_threshold, - ), - ) - .distinct() - ) - result = await db_session.execute(stmt) - publications = list(result.scalars().all()) - - if not publications: - return - - resolved_key = openalex_api_key or settings.openalex_api_key - client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto) - batch_size = 25 - arxiv_lookup_allowed = True - - for i in range(0, len(publications), batch_size): - if await self._run_is_canceled(db_session, run_id=run_id): - logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) - return - batch = publications[i : i + batch_size] - titles = [ - " ".join(re.sub(r"[^\w\s]", " ", p.title_raw).split()) - for p in batch - if p.title_raw and p.title_raw.strip() - ] - - if not titles: - continue - - try: - openalex_works = await client.get_works_by_filter( - {"title.search": "|".join(titles)}, limit=batch_size * 3 - ) - except OpenAlexBudgetExhaustedError: - structured_log( - logger, - "warning", - "ingestion.openalex_budget_exhausted", - run_id=run_id, - ) - break # Stop all enrichment — budget won't reset until midnight UTC - except OpenAlexRateLimitError: - structured_log( - logger, - "warning", - "ingestion.openalex_rate_limited", - run_id=run_id, - ) - await asyncio.sleep(60) - continue - except Exception as e: - structured_log( - logger, - "warning", - "ingestion.openalex_enrichment_failed", - error=str(e), - run_id=run_id, - ) - continue - - for p in batch: - # Check for cancellation periodically within the batch - if await self._run_is_canceled(db_session, run_id=run_id): - logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) - return - - p.openalex_last_attempt_at = now - arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment( - db_session, - publication=p, - run_id=run_id, - allow_arxiv_lookup=arxiv_lookup_allowed, - ) - - match = find_best_match( - target_title=p.title_raw, - target_year=p.year, - target_authors=p.author_text or "", - candidates=openalex_works, - ) - if match: - p.year = match.publication_year or p.year - p.citation_count = match.cited_by_count or p.citation_count - p.pdf_url = match.oa_url or p.pdf_url - p.openalex_enriched = True - - await db_session.flush() - - from app.services.publications.dedup import sweep_identifier_duplicates - - merge_count = await sweep_identifier_duplicates(db_session) - if merge_count: - structured_log( - logger, - "info", - "ingestion.identifier_dedup_sweep", - merged_count=merge_count, - run_id=run_id, - ) - - async def _discover_identifiers_for_enrichment( - self, - db_session: AsyncSession, - *, - publication: Publication, - run_id: int, - allow_arxiv_lookup: bool, - ) -> bool: - if not allow_arxiv_lookup: - await identifier_service.sync_identifiers_for_publication_fields( - db_session, - publication=publication, - ) - await self._publish_identifier_update_event( - db_session, - run_id=run_id, - publication_id=int(publication.id), - ) - return False - try: - await identifier_service.discover_and_sync_identifiers_for_publication( - db_session, - publication=publication, - scholar_label=publication.author_text or "", - ) - await self._publish_identifier_update_event( - db_session, - run_id=run_id, - publication_id=int(publication.id), - ) - return True - except ArxivRateLimitError: - structured_log( - logger, - "warning", - "ingestion.arxiv_rate_limited", - run_id=run_id, - publication_id=int(publication.id), - detail="arXiv temporarily disabled for remaining enrichment pass", - ) - await identifier_service.sync_identifiers_for_publication_fields( - db_session, - publication=publication, - ) - await self._publish_identifier_update_event( - db_session, - run_id=run_id, - publication_id=int(publication.id), - ) - return False - - async def _publish_identifier_update_event( - self, - db_session: AsyncSession, - *, - run_id: int, - publication_id: int, - ) -> None: - display = await identifier_service.display_identifier_for_publication_id( - db_session, - publication_id=publication_id, - ) - if display is None: - return - await run_events.publish( - run_id=run_id, - event_type="identifier_updated", - data={ - "publication_id": int(publication_id), - "display_identifier": { - "kind": display.kind, - "value": display.value, - "label": display.label, - "url": display.url, - "confidence_score": float(display.confidence_score), - }, - }, - ) - - async def _enrich_publications_with_openalex( - self, - scholar: ScholarProfile, - publications: list[PublicationCandidate], - ) -> list[PublicationCandidate]: - if not publications: - return publications - - from app.services.openalex.client import OpenAlexClient - from app.services.openalex.matching import find_best_match - - client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto) - - # Batch requests by 25 to stay within URL length limits - batch_size = 25 - enriched: list[PublicationCandidate] = [] - - import re - - for i in range(0, len(publications), batch_size): - batch = publications[i : i + batch_size] - - titles = [] - for p in batch: - if not p.title: - continue - # Strip all non-alphanumeric words to prevent OpenAlex API injection crashes - # and minimize punctuation-based duplicate misses. - safe_title = re.sub(r"[^\w\s]", " ", p.title) - safe_title = " ".join(safe_title.split()) - if safe_title: - titles.append(safe_title) - - if not titles: - enriched.extend(batch) - continue - - query = "|".join(t for t in titles) - try: - openalex_works = await client.get_works_by_filter({"title.search": query}, limit=batch_size * 3) - except Exception as e: - logger.warning( - "ingestion.openalex_enrichment_failed", - extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id}, - ) - openalex_works = [] - - for p in batch: - match = find_best_match( - target_title=p.title, - target_year=p.year, - target_authors=p.authors_text or (scholar.display_name or scholar.scholar_id), - candidates=openalex_works, - ) - if match: - new_p = PublicationCandidate( - title=p.title, - title_url=p.title_url, - cluster_id=p.cluster_id, - year=match.publication_year or p.year, - citation_count=match.cited_by_count, - authors_text=p.authors_text, - venue_text=p.venue_text, - pdf_url=match.oa_url or p.pdf_url, - ) - enriched.append(new_p) - else: - enriched.append(p) - return enriched - - def _resolve_run_status( - self, - *, - scholar_count: int, - succeeded_count: int, - failed_count: int, - partial_count: int, - ) -> RunStatus: - if scholar_count == 0: - return RunStatus.SUCCESS - if failed_count == scholar_count: - return RunStatus.FAILED - if failed_count > 0 or partial_count > 0: - return RunStatus.PARTIAL_FAILURE - if succeeded_count > 0: - return RunStatus.SUCCESS - return RunStatus.FAILED - - def _resolve_continuation_queue_target( - self, - *, - outcome: str, - state: str, - pagination_truncated_reason: str | None, - continuation_cstart: int | None, - fallback_cstart: int, - ) -> tuple[str | None, int]: - if outcome == "partial": - reason = (pagination_truncated_reason or "").strip() - if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(RESUMABLE_PARTIAL_REASON_PREFIXES): - return reason, queue_service.normalize_cstart( - continuation_cstart if continuation_cstart is not None else fallback_cstart - ) - return None, queue_service.normalize_cstart(fallback_cstart) - - if outcome == "failed" and state == ParseState.NETWORK_ERROR.value: - return "network_error_retry", queue_service.normalize_cstart( - continuation_cstart if continuation_cstart is not None else fallback_cstart - ) - - return None, queue_service.normalize_cstart(fallback_cstart) - - def _build_failure_debug_context( - self, - *, - fetch_result: FetchResult, - parsed_page: ParsedProfilePage, - attempt_log: list[dict[str, Any]], - page_logs: list[dict[str, Any]] | None = None, - exception: Exception | None = None, - ) -> dict[str, Any]: - context: dict[str, Any] = { - "requested_url": fetch_result.requested_url, - "final_url": fetch_result.final_url, - "status_code": fetch_result.status_code, - "fetch_error": fetch_result.error, - "state_reason": parsed_page.state_reason, - "profile_name": parsed_page.profile_name, - "profile_image_url": parsed_page.profile_image_url, - "articles_range": parsed_page.articles_range, - "has_show_more_button": parsed_page.has_show_more_button, - "has_operation_error_banner": parsed_page.has_operation_error_banner, - "warning_codes": parsed_page.warnings, - "marker_counts_nonzero": {key: value for key, value in parsed_page.marker_counts.items() if value > 0}, - "body_length": len(fetch_result.body), - "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() if fetch_result.body else None, - "body_excerpt": _build_body_excerpt(fetch_result.body), - "attempt_log": attempt_log, - } - if page_logs: - context["page_logs"] = page_logs - if exception is not None: - context["exception_type"] = type(exception).__name__ - context["exception_message"] = str(exception) - return context + return run_execution_summary(run=run, scholars=scholars, progress=progress) async def _try_acquire_user_lock( self, @@ -2076,9 +630,6 @@ class ScholarIngestionService: ) -> bool: result = await db_session.execute( text("SELECT pg_try_advisory_xact_lock(:namespace, :user_key)"), - { - "namespace": RUN_LOCK_NAMESPACE, - "user_key": int(user_id), - }, + {"namespace": RUN_LOCK_NAMESPACE, "user_key": int(user_id)}, ) return bool(result.scalar_one()) diff --git a/app/services/ingestion/enrichment.py b/app/services/ingestion/enrichment.py new file mode 100644 index 0000000..e63a6aa --- /dev/null +++ b/app/services/ingestion/enrichment.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +import asyncio +import logging +import re +from datetime import UTC, datetime, timedelta + +from sqlalchemy import or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + Publication, + RunStatus, + ScholarProfile, + ScholarPublication, +) +from app.logging_utils import structured_log +from app.services.arxiv.errors import ArxivRateLimitError +from app.services.publication_identifiers import application as identifier_service +from app.services.runs.events import run_events +from app.services.scholar.parser import PublicationCandidate +from app.settings import settings + +logger = logging.getLogger(__name__) + + +class EnrichmentRunner: + """Post-run OpenAlex enrichment logic. + + Receives service dependencies at construction so it can be tested + independently of ``ScholarIngestionService``. + """ + + async def run_is_canceled( + self, + db_session: AsyncSession, + *, + run_id: int, + ) -> bool: + check_result = await db_session.execute(select(CrawlRun.status).where(CrawlRun.id == run_id)) + status = check_result.scalar_one_or_none() + if status is None: + raise RuntimeError(f"Missing crawl_run for run_id={run_id}.") + return status == RunStatus.CANCELED + + async def enrich_pending_publications( + self, + db_session: AsyncSession, + *, + run_id: int, + openalex_api_key: str | None = None, + ) -> None: + """Enrich unenriched publications with OpenAlex data. + + Stops immediately on budget exhaustion (429 with $0 remaining). + Sleeps 60s and continues on transient rate limits. + """ + from app.services.openalex.client import ( + OpenAlexBudgetExhaustedError, + OpenAlexClient, + OpenAlexRateLimitError, + ) + from app.services.openalex.matching import find_best_match + + run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id)) + user_id = run_result.scalar_one() + + now = datetime.now(UTC) + cooldown_threshold = now - timedelta(days=7) + + stmt = ( + select(Publication) + .join(ScholarPublication) + .join(ScholarProfile, ScholarPublication.scholar_profile_id == ScholarProfile.id) + .where( + ScholarProfile.user_id == user_id, + Publication.openalex_enriched.is_(False), + or_( + Publication.openalex_last_attempt_at.is_(None), + Publication.openalex_last_attempt_at < cooldown_threshold, + ), + ) + .distinct() + ) + result = await db_session.execute(stmt) + publications = list(result.scalars().all()) + + if not publications: + return + + resolved_key = openalex_api_key or settings.openalex_api_key + client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto) + batch_size = 25 + arxiv_lookup_allowed = True + + for i in range(0, len(publications), batch_size): + if await self.run_is_canceled(db_session, run_id=run_id): + logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) + return + batch = publications[i : i + batch_size] + titles = [ + " ".join(re.sub(r"[^\w\s]", " ", p.title_raw).split()) + for p in batch + if p.title_raw and p.title_raw.strip() + ] + + if not titles: + continue + + try: + openalex_works = await client.get_works_by_filter( + {"title.search": "|".join(titles)}, limit=batch_size * 3 + ) + except OpenAlexBudgetExhaustedError: + structured_log( + logger, + "warning", + "ingestion.openalex_budget_exhausted", + run_id=run_id, + ) + break + except OpenAlexRateLimitError: + structured_log( + logger, + "warning", + "ingestion.openalex_rate_limited", + run_id=run_id, + ) + await asyncio.sleep(60) + continue + except Exception as e: + structured_log( + logger, + "warning", + "ingestion.openalex_enrichment_failed", + error=str(e), + run_id=run_id, + ) + continue + + for p in batch: + if await self.run_is_canceled(db_session, run_id=run_id): + logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id}) + return + + p.openalex_last_attempt_at = now + arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment( + db_session, + publication=p, + run_id=run_id, + allow_arxiv_lookup=arxiv_lookup_allowed, + ) + + match = find_best_match( + target_title=p.title_raw, + target_year=p.year, + target_authors=p.author_text or "", + candidates=openalex_works, + ) + if match: + p.year = match.publication_year or p.year + p.citation_count = match.cited_by_count or p.citation_count + p.pdf_url = match.oa_url or p.pdf_url + p.openalex_enriched = True + + await db_session.flush() + + from app.services.publications.dedup import sweep_identifier_duplicates + + merge_count = await sweep_identifier_duplicates(db_session) + if merge_count: + structured_log( + logger, + "info", + "ingestion.identifier_dedup_sweep", + merged_count=merge_count, + run_id=run_id, + ) + + async def _discover_identifiers_for_enrichment( + self, + db_session: AsyncSession, + *, + publication: Publication, + run_id: int, + allow_arxiv_lookup: bool, + ) -> bool: + if not allow_arxiv_lookup: + await identifier_service.sync_identifiers_for_publication_fields( + db_session, + publication=publication, + ) + await self._publish_identifier_update_event( + db_session, + run_id=run_id, + publication_id=int(publication.id), + ) + return False + try: + await identifier_service.discover_and_sync_identifiers_for_publication( + db_session, + publication=publication, + scholar_label=publication.author_text or "", + ) + await self._publish_identifier_update_event( + db_session, + run_id=run_id, + publication_id=int(publication.id), + ) + return True + except ArxivRateLimitError: + structured_log( + logger, + "warning", + "ingestion.arxiv_rate_limited", + run_id=run_id, + publication_id=int(publication.id), + detail="arXiv temporarily disabled for remaining enrichment pass", + ) + await identifier_service.sync_identifiers_for_publication_fields( + db_session, + publication=publication, + ) + await self._publish_identifier_update_event( + db_session, + run_id=run_id, + publication_id=int(publication.id), + ) + return False + + async def _publish_identifier_update_event( + self, + db_session: AsyncSession, + *, + run_id: int, + publication_id: int, + ) -> None: + display = await identifier_service.display_identifier_for_publication_id( + db_session, + publication_id=publication_id, + ) + if display is None: + return + await run_events.publish( + run_id=run_id, + event_type="identifier_updated", + data={ + "publication_id": int(publication_id), + "display_identifier": { + "kind": display.kind, + "value": display.value, + "label": display.label, + "url": display.url, + "confidence_score": float(display.confidence_score), + }, + }, + ) + + async def enrich_publications_with_openalex( + self, + scholar: ScholarProfile, + publications: list[PublicationCandidate], + ) -> list[PublicationCandidate]: + if not publications: + return publications + + from app.services.openalex.client import OpenAlexClient + from app.services.openalex.matching import find_best_match + + client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto) + + batch_size = 25 + enriched: list[PublicationCandidate] = [] + + for i in range(0, len(publications), batch_size): + batch = publications[i : i + batch_size] + + titles = [] + for p in batch: + if not p.title: + continue + safe_title = re.sub(r"[^\w\s]", " ", p.title) + safe_title = " ".join(safe_title.split()) + if safe_title: + titles.append(safe_title) + + if not titles: + enriched.extend(batch) + continue + + query = "|".join(t for t in titles) + try: + openalex_works = await client.get_works_by_filter({"title.search": query}, limit=batch_size * 3) + except Exception as e: + logger.warning( + "ingestion.openalex_enrichment_failed", + extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id}, + ) + openalex_works = [] + + for p in batch: + match = find_best_match( + target_title=p.title, + target_year=p.year, + target_authors=p.authors_text or (scholar.display_name or scholar.scholar_id), + candidates=openalex_works, + ) + if match: + new_p = PublicationCandidate( + title=p.title, + title_url=p.title_url, + cluster_id=p.cluster_id, + year=match.publication_year or p.year, + citation_count=match.cited_by_count, + authors_text=p.authors_text, + venue_text=p.venue_text, + pdf_url=match.oa_url or p.pdf_url, + ) + enriched.append(new_p) + else: + enriched.append(p) + return enriched diff --git a/app/services/ingestion/run_completion.py b/app/services/ingestion/run_completion.py new file mode 100644 index 0000000..450550d --- /dev/null +++ b/app/services/ingestion/run_completion.py @@ -0,0 +1,417 @@ +from __future__ import annotations + +import hashlib +import logging +from datetime import UTC, datetime +from typing import Any + +from app.db.models import ( + CrawlRun, + RunStatus, + ScholarProfile, +) +from app.logging_utils import structured_log +from app.services.ingestion import safety as run_safety_service +from app.services.ingestion.constants import ( + FAILED_STATES, + FAILURE_BUCKET_BLOCKED, + FAILURE_BUCKET_INGESTION, + FAILURE_BUCKET_LAYOUT, + FAILURE_BUCKET_NETWORK, + FAILURE_BUCKET_OTHER, +) +from app.services.ingestion.fingerprints import _build_body_excerpt +from app.services.ingestion.types import ( + RunAlertSummary, + RunExecutionSummary, + RunFailureSummary, + RunProgress, + ScholarProcessingOutcome, +) +from app.services.scholar.parser import ParsedProfilePage, ParseState +from app.services.scholar.source import FetchResult +from app.settings import settings + +logger = logging.getLogger(__name__) + + +def _int_or_default(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def classify_failure_bucket(*, state: str, state_reason: str) -> str: + reason = state_reason.strip().lower() + normalized_state = state.strip().lower() + + if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"): + return FAILURE_BUCKET_BLOCKED + if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"): + return FAILURE_BUCKET_NETWORK + if normalized_state == ParseState.LAYOUT_CHANGED.value: + return FAILURE_BUCKET_LAYOUT + if normalized_state == "ingestion_error": + return FAILURE_BUCKET_INGESTION + return FAILURE_BUCKET_OTHER + + +def summarize_failures( + *, + scholar_results: list[dict[str, Any]], +) -> RunFailureSummary: + failed_state_counts: dict[str, int] = {} + failed_reason_counts: dict[str, int] = {} + scrape_failure_counts: dict[str, int] = {} + retries_scheduled_count = 0 + scholars_with_retries_count = 0 + retry_exhausted_count = 0 + for entry in scholar_results: + retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1) + if retries_for_entry > 0: + retries_scheduled_count += retries_for_entry + scholars_with_retries_count += 1 + if str(entry.get("outcome", "")) != "failed": + continue + state = str(entry.get("state", "")).strip() + if state not in FAILED_STATES: + continue + failed_state_counts[state] = failed_state_counts.get(state, 0) + 1 + reason = str(entry.get("state_reason", "")).strip() + if reason: + failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1 + bucket = classify_failure_bucket(state=state, state_reason=reason) + scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1 + if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0: + retry_exhausted_count += 1 + return RunFailureSummary( + failed_state_counts=failed_state_counts, + failed_reason_counts=failed_reason_counts, + scrape_failure_counts=scrape_failure_counts, + retries_scheduled_count=retries_scheduled_count, + scholars_with_retries_count=scholars_with_retries_count, + retry_exhausted_count=retry_exhausted_count, + ) + + +def build_alert_summary( + *, + failure_summary: RunFailureSummary, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, +) -> RunAlertSummary: + blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0)) + network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0)) + blocked_threshold = max(1, int(alert_blocked_failure_threshold)) + network_threshold = max(1, int(alert_network_failure_threshold)) + retry_threshold = max(1, int(alert_retry_scheduled_threshold)) + alert_flags = { + "blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold, + "network_failure_threshold_exceeded": network_failure_count >= network_threshold, + "retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold, + } + return RunAlertSummary( + blocked_failure_count=blocked_failure_count, + network_failure_count=network_failure_count, + blocked_failure_threshold=blocked_threshold, + network_failure_threshold=network_threshold, + retry_scheduled_threshold=retry_threshold, + alert_flags=alert_flags, + ) + + +def apply_safety_outcome( + *, + user_settings: Any, + run: CrawlRun, + user_id: int, + alert_summary: RunAlertSummary, +) -> None: + pre_apply_state = run_safety_service.get_safety_event_context( + user_settings, + now_utc=datetime.now(UTC), + ) + safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome( + user_settings, + run_id=int(run.id), + blocked_failure_count=alert_summary.blocked_failure_count, + network_failure_count=alert_summary.network_failure_count, + blocked_failure_threshold=alert_summary.blocked_failure_threshold, + network_failure_threshold=alert_summary.network_failure_threshold, + blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds, + network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds, + now_utc=datetime.now(UTC), + ) + if cooldown_trigger_reason is not None: + structured_log( + logger, + "warning", + "ingestion.safety_cooldown_entered", + user_id=user_id, + crawl_run_id=int(run.id), + reason=cooldown_trigger_reason, + blocked_failure_count=alert_summary.blocked_failure_count, + network_failure_count=alert_summary.network_failure_count, + blocked_failure_threshold=alert_summary.blocked_failure_threshold, + network_failure_threshold=alert_summary.network_failure_threshold, + cooldown_until=safety_state.get("cooldown_until"), + cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"), + safety_counters=safety_state.get("counters", {}), + ) + elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"): + structured_log( + logger, + "info", + "ingestion.cooldown_cleared", + user_id=user_id, + crawl_run_id=int(run.id), + reason=pre_apply_state.get("cooldown_reason"), + cooldown_until=pre_apply_state.get("cooldown_until"), + ) + + +def finalize_run_record( + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, + failure_summary: RunFailureSummary, + alert_summary: RunAlertSummary, + idempotency_key: str | None, + run_status: RunStatus, +) -> None: + run.end_dt = datetime.now(UTC) + if run.status != RunStatus.CANCELED: + run.status = run_status + run.error_log = { + "scholar_results": progress.scholar_results, + "summary": { + "succeeded_count": progress.succeeded_count, + "failed_count": progress.failed_count, + "partial_count": progress.partial_count, + "failed_state_counts": failure_summary.failed_state_counts, + "failed_reason_counts": failure_summary.failed_reason_counts, + "scrape_failure_counts": failure_summary.scrape_failure_counts, + "retry_counts": { + "retries_scheduled_count": failure_summary.retries_scheduled_count, + "scholars_with_retries_count": failure_summary.scholars_with_retries_count, + "retry_exhausted_count": failure_summary.retry_exhausted_count, + }, + "alert_thresholds": { + "blocked_failure_threshold": alert_summary.blocked_failure_threshold, + "network_failure_threshold": alert_summary.network_failure_threshold, + "retry_scheduled_threshold": alert_summary.retry_scheduled_threshold, + }, + "alert_flags": alert_summary.alert_flags, + }, + "meta": {"idempotency_key": idempotency_key} if idempotency_key else {}, + } + + +def resolve_run_status( + *, + scholar_count: int, + succeeded_count: int, + failed_count: int, + partial_count: int, +) -> RunStatus: + if scholar_count == 0: + return RunStatus.SUCCESS + if failed_count == scholar_count: + return RunStatus.FAILED + if failed_count > 0 or partial_count > 0: + return RunStatus.PARTIAL_FAILURE + if succeeded_count > 0: + return RunStatus.SUCCESS + return RunStatus.FAILED + + +def build_failure_debug_context( + *, + fetch_result: FetchResult, + parsed_page: ParsedProfilePage, + attempt_log: list[dict[str, Any]], + page_logs: list[dict[str, Any]] | None = None, + exception: Exception | None = None, +) -> dict[str, Any]: + context: dict[str, Any] = { + "requested_url": fetch_result.requested_url, + "final_url": fetch_result.final_url, + "status_code": fetch_result.status_code, + "fetch_error": fetch_result.error, + "state_reason": parsed_page.state_reason, + "profile_name": parsed_page.profile_name, + "profile_image_url": parsed_page.profile_image_url, + "articles_range": parsed_page.articles_range, + "has_show_more_button": parsed_page.has_show_more_button, + "has_operation_error_banner": parsed_page.has_operation_error_banner, + "warning_codes": parsed_page.warnings, + "marker_counts_nonzero": {key: value for key, value in parsed_page.marker_counts.items() if value > 0}, + "body_length": len(fetch_result.body), + "body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() if fetch_result.body else None, + "body_excerpt": _build_body_excerpt(fetch_result.body), + "attempt_log": attempt_log, + } + if page_logs: + context["page_logs"] = page_logs + if exception is not None: + context["exception_type"] = type(exception).__name__ + context["exception_message"] = str(exception) + return context + + +def complete_run_for_user( + *, + user_settings: Any, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + progress: RunProgress, + idempotency_key: str | None, + alert_blocked_failure_threshold: int, + alert_network_failure_threshold: int, + alert_retry_scheduled_threshold: int, +) -> tuple[RunFailureSummary, RunAlertSummary]: + failure_summary = summarize_failures(scholar_results=progress.scholar_results) + alert_summary = build_alert_summary( + failure_summary=failure_summary, + alert_blocked_failure_threshold=alert_blocked_failure_threshold, + alert_network_failure_threshold=alert_network_failure_threshold, + alert_retry_scheduled_threshold=alert_retry_scheduled_threshold, + ) + if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]: + structured_log( + logger, + "warning", + "ingestion.alert_blocked_failure_threshold_exceeded", + user_id=user_id, + crawl_run_id=int(run.id), + blocked_failure_count=alert_summary.blocked_failure_count, + threshold=alert_summary.blocked_failure_threshold, + ) + if alert_summary.alert_flags["network_failure_threshold_exceeded"]: + structured_log( + logger, + "warning", + "ingestion.alert_network_failure_threshold_exceeded", + user_id=user_id, + crawl_run_id=int(run.id), + network_failure_count=alert_summary.network_failure_count, + threshold=alert_summary.network_failure_threshold, + ) + if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]: + structured_log( + logger, + "warning", + "ingestion.alert_retry_scheduled_threshold_exceeded", + user_id=user_id, + crawl_run_id=int(run.id), + retries_scheduled_count=failure_summary.retries_scheduled_count, + threshold=alert_summary.retry_scheduled_threshold, + ) + apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary) + run_status = resolve_run_status( + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + ) + finalize_run_record( + run=run, + scholars=scholars, + progress=progress, + failure_summary=failure_summary, + alert_summary=alert_summary, + idempotency_key=idempotency_key, + run_status=run_status, + ) + return failure_summary, alert_summary + + +def result_counters(result_entry: dict[str, Any]) -> tuple[int, int, int]: + outcome = str(result_entry.get("outcome", "")).strip().lower() + if outcome == "success": + return 1, 0, 0 + if outcome == "partial": + return 1, 0, 1 + if outcome == "failed": + return 0, 1, 0 + raise RuntimeError(f"Unexpected scholar outcome label: {outcome!r}") + + +def find_scholar_result_index( + *, + scholar_results: list[dict[str, Any]], + scholar_profile_id: int, +) -> int | None: + for index, result_entry in enumerate(scholar_results): + current_scholar_id = _int_or_default(result_entry.get("scholar_profile_id"), 0) + if current_scholar_id == scholar_profile_id: + return index + return None + + +def adjust_progress_counts( + *, + progress: RunProgress, + succeeded_delta: int, + failed_delta: int, + partial_delta: int, +) -> None: + progress.succeeded_count += succeeded_delta + progress.failed_count += failed_delta + progress.partial_count += partial_delta + if progress.succeeded_count < 0 or progress.failed_count < 0 or progress.partial_count < 0: + raise RuntimeError("RunProgress counters entered invalid negative state.") + + +def apply_outcome_to_progress( + *, + progress: RunProgress, + outcome: ScholarProcessingOutcome, +) -> None: + scholar_profile_id = _int_or_default(outcome.result_entry.get("scholar_profile_id"), 0) + if scholar_profile_id <= 0: + raise RuntimeError("Scholar outcome missing valid scholar_profile_id.") + prior_index = find_scholar_result_index( + scholar_results=progress.scholar_results, + scholar_profile_id=scholar_profile_id, + ) + next_succeeded, next_failed, next_partial = result_counters(outcome.result_entry) + if prior_index is None: + progress.scholar_results.append(outcome.result_entry) + adjust_progress_counts( + progress=progress, + succeeded_delta=next_succeeded, + failed_delta=next_failed, + partial_delta=next_partial, + ) + return + previous_entry = progress.scholar_results[prior_index] + prev_succeeded, prev_failed, prev_partial = result_counters(previous_entry) + progress.scholar_results[prior_index] = outcome.result_entry + adjust_progress_counts( + progress=progress, + succeeded_delta=next_succeeded - prev_succeeded, + failed_delta=next_failed - prev_failed, + partial_delta=next_partial - prev_partial, + ) + + +def run_execution_summary( + *, + run: CrawlRun, + scholars: list[ScholarProfile], + progress: RunProgress, +) -> RunExecutionSummary: + return RunExecutionSummary( + crawl_run_id=run.id, + status=run.status, + scholar_count=len(scholars), + succeeded_count=progress.succeeded_count, + failed_count=progress.failed_count, + partial_count=progress.partial_count, + new_publication_count=run.new_pub_count, + ) diff --git a/app/services/ingestion/scholar_processing.py b/app/services/ingestion/scholar_processing.py new file mode 100644 index 0000000..393f85c --- /dev/null +++ b/app/services/ingestion/scholar_processing.py @@ -0,0 +1,614 @@ +from __future__ import annotations + +import asyncio +import logging +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import ( + CrawlRun, + RunStatus, + ScholarProfile, +) +from app.logging_utils import structured_log +from app.services.ingestion import queue as queue_service +from app.services.ingestion.constants import ( + RESUMABLE_PARTIAL_REASON_PREFIXES, + RESUMABLE_PARTIAL_REASONS, +) +from app.services.ingestion.pagination import PaginationEngine +from app.services.ingestion.publication_upsert import upsert_profile_publications +from app.services.ingestion.run_completion import apply_outcome_to_progress, build_failure_debug_context +from app.services.ingestion.types import ( + PagedParseResult, + RunProgress, + ScholarProcessingOutcome, +) +from app.services.scholar.parser import ParseState + +logger = logging.getLogger(__name__) + + +def assert_valid_paged_parse_result( + *, + scholar_id: str, + paged_parse_result: PagedParseResult, +) -> None: + parsed_page = paged_parse_result.parsed_page + if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any( + code.startswith("layout_") for code in parsed_page.warnings + ): + raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.") + for publication in paged_parse_result.publications: + if not publication.title.strip(): + raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.") + if publication.citation_count is not None and int(publication.citation_count) < 0: + raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.") + + +def apply_first_page_profile_metadata( + *, + scholar: ScholarProfile, + paged_parse_result: PagedParseResult, + run_dt: datetime, +) -> None: + first_page = paged_parse_result.first_page_parsed_page + if first_page.profile_name and not (scholar.display_name or "").strip(): + scholar.display_name = first_page.profile_name + if first_page.profile_image_url: + scholar.profile_image_url = first_page.profile_image_url + scholar.last_initial_page_checked_at = run_dt + + +def build_result_entry( + *, + scholar: ScholarProfile, + start_cstart: int, + paged_parse_result: PagedParseResult, +) -> dict[str, Any]: + parsed_page = paged_parse_result.parsed_page + return { + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": parsed_page.state.value, + "state_reason": parsed_page.state_reason, + "outcome": "failed", + "attempt_count": len(paged_parse_result.attempt_log), + "publication_count": len(paged_parse_result.publications), + "start_cstart": start_cstart, + "articles_range": parsed_page.articles_range, + "warnings": parsed_page.warnings, + "has_show_more_button": parsed_page.has_show_more_button, + "pages_fetched": paged_parse_result.pages_fetched, + "pages_attempted": paged_parse_result.pages_attempted, + "has_more_remaining": paged_parse_result.has_more_remaining, + "pagination_truncated_reason": paged_parse_result.pagination_truncated_reason, + "continuation_cstart": paged_parse_result.continuation_cstart, + "skipped_no_change": paged_parse_result.skipped_no_change, + "initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + } + + +def skipped_no_change_outcome( + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], +) -> ScholarProcessingOutcome: + first_page = paged_parse_result.first_page_parsed_page + scholar.last_run_status = RunStatus.SUCCESS + scholar.last_run_dt = run_dt + result_entry["state"] = first_page.state.value + result_entry["state_reason"] = "no_change_initial_page_signature" + result_entry["outcome"] = "success" + result_entry["publication_count"] = 0 + result_entry["warnings"] = first_page.warnings + result_entry["debug"] = { + "state_reason": "no_change_initial_page_signature", + "first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256, + "attempt_log": paged_parse_result.attempt_log, + "page_logs": paged_parse_result.page_logs, + } + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=0, + discovered_publication_count=0, + ) + + +async def upsert_publications_outcome( + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], +) -> ScholarProcessingOutcome: + parsed_page = paged_parse_result.parsed_page + publications = paged_parse_result.publications + had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS} + has_partial_set = len(publications) > 0 and had_page_failure + if (not had_page_failure) or has_partial_set: + return await _upsert_success_or_exception( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_set, + ) + return _parse_failure_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + +async def _upsert_success_or_exception( + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, +) -> ScholarProcessingOutcome: + try: + return _upsert_success( + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + has_partial_publication_set=has_partial_publication_set, + ) + except Exception as exc: + return _upsert_exception_outcome( + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + exc=exc, + ) + + +def _upsert_success( + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + has_partial_publication_set: bool, +) -> ScholarProcessingOutcome: + discovered_count = paged_parse_result.discovered_publication_count + is_partial = ( + paged_parse_result.has_more_remaining + or paged_parse_result.pagination_truncated_reason is not None + or has_partial_publication_set + ) + scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS + scholar.last_run_dt = run_dt + if not is_partial and paged_parse_result.first_page_fingerprint_sha256: + scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256 + result_entry["outcome"] = "partial" if is_partial else "success" + if is_partial: + result_entry["debug"] = build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + return ScholarProcessingOutcome( + result_entry=result_entry, + succeeded_count_delta=1, + failed_count_delta=0, + partial_count_delta=1 if is_partial else 0, + discovered_publication_count=discovered_count, + ) + + +def _upsert_exception_outcome( + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], + exc: Exception, +) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["state"] = "ingestion_error" + result_entry["state_reason"] = "publication_upsert_exception" + result_entry["outcome"] = "failed" + result_entry["error"] = str(exc) + result_entry["debug"] = build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + exception=exc, + ) + logger.exception( + "ingestion.scholar_failed", + extra={ + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + +def _parse_failure_outcome( + *, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], +) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = run_dt + result_entry["debug"] = build_failure_debug_context( + fetch_result=paged_parse_result.fetch_result, + parsed_page=paged_parse_result.parsed_page, + attempt_log=paged_parse_result.attempt_log, + page_logs=paged_parse_result.page_logs, + ) + structured_log( + logger, + "warning", + "ingestion.scholar_parse_failed", + scholar_profile_id=scholar.id, + scholar_id=scholar.scholar_id, + state=paged_parse_result.parsed_page.state.value, + state_reason=paged_parse_result.parsed_page.state_reason, + status_code=paged_parse_result.fetch_result.status_code, + ) + return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0) + + +async def sync_continuation_queue( + db_session: AsyncSession, + *, + user_id: int, + scholar: ScholarProfile, + run: CrawlRun, + start_cstart: int, + result_entry: dict[str, Any], + paged_parse_result: PagedParseResult, + auto_queue_continuations: bool, + queue_delay_seconds: int, +) -> None: + queue_reason, queue_cstart = resolve_continuation_queue_target( + outcome=str(result_entry.get("outcome", "")), + state=str(result_entry.get("state", "")), + pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, + continuation_cstart=paged_parse_result.continuation_cstart, + fallback_cstart=start_cstart, + ) + if auto_queue_continuations and queue_reason is not None: + await queue_service.upsert_job( + db_session, + user_id=user_id, + scholar_profile_id=scholar.id, + resume_cstart=queue_cstart, + reason=queue_reason, + run_id=run.id, + delay_seconds=queue_delay_seconds, + ) + result_entry["continuation_enqueued"] = True + result_entry["continuation_reason"] = queue_reason + result_entry["continuation_cstart"] = queue_cstart + return + if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id): + result_entry["continuation_cleared"] = True + + +def resolve_continuation_queue_target( + *, + outcome: str, + state: str, + pagination_truncated_reason: str | None, + continuation_cstart: int | None, + fallback_cstart: int, +) -> tuple[str | None, int]: + if outcome == "partial": + reason = (pagination_truncated_reason or "").strip() + if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(RESUMABLE_PARTIAL_REASON_PREFIXES): + return reason, queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + return None, queue_service.normalize_cstart(fallback_cstart) + + if outcome == "failed" and state == ParseState.NETWORK_ERROR.value: + return "network_error_retry", queue_service.normalize_cstart( + continuation_cstart if continuation_cstart is not None else fallback_cstart + ) + + return None, queue_service.normalize_cstart(fallback_cstart) + + +async def process_scholar( + db_session: AsyncSession, + *, + pagination: PaginationEngine, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + start_cstart: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, +) -> ScholarProcessingOutcome: + try: + run_dt, paged_parse_result, result_entry = await _fetch_and_prepare_scholar_result( + db_session, + pagination=pagination, + run=run, + scholar=scholar, + user_id=user_id, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages_per_scholar=max_pages_per_scholar, + page_size=page_size, + ) + outcome = await _resolve_scholar_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + await sync_continuation_queue( + db_session, + user_id=user_id, + scholar=scholar, + run=run, + start_cstart=start_cstart, + result_entry=outcome.result_entry, + paged_parse_result=paged_parse_result, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + ) + return outcome + except Exception as exc: + return unexpected_scholar_exception_outcome( + run=run, + scholar=scholar, + start_cstart=start_cstart, + exc=exc, + ) + + +async def _fetch_and_prepare_scholar_result( + db_session: AsyncSession, + *, + pagination: PaginationEngine, + run: CrawlRun, + scholar: ScholarProfile, + user_id: int, + start_cstart: int, + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, +) -> tuple[datetime, PagedParseResult, dict[str, Any]]: + run_dt = datetime.now(UTC) + paged_parse_result = await pagination.fetch_and_parse_all_pages( + scholar=scholar, + run=run, + db_session=db_session, + start_cstart=start_cstart, + request_delay_seconds=request_delay_seconds, + network_error_retries=network_error_retries, + retry_backoff_seconds=retry_backoff_seconds, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff_seconds=rate_limit_backoff_seconds, + max_pages=max_pages_per_scholar, + page_size=page_size, + previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256, + upsert_publications_fn=upsert_profile_publications, + ) + assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result) + apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt) + parsed_page = paged_parse_result.parsed_page + structured_log( + logger, + "info", + "ingestion.scholar_parsed", + user_id=user_id, + crawl_run_id=run.id, + scholar_profile_id=scholar.id, + scholar_id=scholar.scholar_id, + state=parsed_page.state.value, + publication_count=len(paged_parse_result.publications), + has_show_more_button=parsed_page.has_show_more_button, + pages_fetched=paged_parse_result.pages_fetched, + pages_attempted=paged_parse_result.pages_attempted, + has_more_remaining=paged_parse_result.has_more_remaining, + pagination_truncated_reason=paged_parse_result.pagination_truncated_reason, + warning_count=len(parsed_page.warnings), + skipped_no_change=paged_parse_result.skipped_no_change, + ) + result_entry = build_result_entry( + scholar=scholar, + start_cstart=start_cstart, + paged_parse_result=paged_parse_result, + ) + return run_dt, paged_parse_result, result_entry + + +async def _resolve_scholar_outcome( + db_session: AsyncSession, + *, + run: CrawlRun, + scholar: ScholarProfile, + run_dt: datetime, + paged_parse_result: PagedParseResult, + result_entry: dict[str, Any], +) -> ScholarProcessingOutcome: + if paged_parse_result.skipped_no_change: + return skipped_no_change_outcome( + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + return await upsert_publications_outcome( + db_session, + run=run, + scholar=scholar, + run_dt=run_dt, + paged_parse_result=paged_parse_result, + result_entry=result_entry, + ) + + +def unexpected_scholar_exception_outcome( + *, + run: CrawlRun, + scholar: ScholarProfile, + start_cstart: int, + exc: Exception, +) -> ScholarProcessingOutcome: + scholar.last_run_status = RunStatus.FAILED + scholar.last_run_dt = datetime.now(UTC) + logger.exception( + "ingestion.scholar_unexpected_failure", + extra={ + "crawl_run_id": run.id, + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + }, + ) + return ScholarProcessingOutcome( + result_entry={ + "scholar_profile_id": scholar.id, + "scholar_id": scholar.scholar_id, + "state": "ingestion_error", + "state_reason": "scholar_processing_exception", + "outcome": "failed", + "attempt_count": 0, + "publication_count": 0, + "start_cstart": start_cstart, + "warnings": [], + "error": str(exc), + "debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)}, + }, + succeeded_count_delta=0, + failed_count_delta=1, + partial_count_delta=0, + discovered_publication_count=0, + ) + + +async def run_scholar_iteration( + db_session: AsyncSession, + *, + pagination: PaginationEngine, + run: CrawlRun, + scholars: list[ScholarProfile], + user_id: int, + start_cstart_map: dict[int, int], + request_delay_seconds: int, + network_error_retries: int, + retry_backoff_seconds: float, + rate_limit_retries: int, + rate_limit_backoff_seconds: float, + max_pages_per_scholar: int, + page_size: int, + auto_queue_continuations: bool, + queue_delay_seconds: int, +) -> RunProgress: + progress = RunProgress() + scholar_kwargs = { + "request_delay_seconds": request_delay_seconds, + "network_error_retries": network_error_retries, + "retry_backoff_seconds": retry_backoff_seconds, + "rate_limit_retries": rate_limit_retries, + "rate_limit_backoff_seconds": rate_limit_backoff_seconds, + "page_size": page_size, + } + + # ── Pass 1: first page of every scholar (breadth-first) ────────── + first_pass_cstarts: dict[int, int] = {} + for index, scholar in enumerate(scholars): + await db_session.refresh(run) + if run.status == RunStatus.CANCELED: + structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id) + return progress + if index > 0 and request_delay_seconds > 0: + await asyncio.sleep(float(request_delay_seconds)) + start_cstart = int(start_cstart_map.get(int(scholar.id), 0)) + outcome = await process_scholar( + db_session, + pagination=pagination, + run=run, + scholar=scholar, + user_id=user_id, + start_cstart=start_cstart, + max_pages_per_scholar=1, + auto_queue_continuations=False, + queue_delay_seconds=queue_delay_seconds, + **scholar_kwargs, + ) + apply_outcome_to_progress(progress=progress, outcome=outcome) + resume_cstart = outcome.result_entry.get("continuation_cstart") + if resume_cstart is not None and int(resume_cstart) > start_cstart: + first_pass_cstarts[int(scholar.id)] = int(resume_cstart) + + # ── Pass 2: remaining pages for each scholar (depth) ───────────── + remaining_max = max(max_pages_per_scholar - 1, 0) + if remaining_max <= 0: + return progress + + for index, scholar in enumerate(scholars): + resume_cstart = first_pass_cstarts.get(int(scholar.id)) + if resume_cstart is None: + continue + await db_session.refresh(run) + if run.status == RunStatus.CANCELED: + structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id) + break + if index > 0 and request_delay_seconds > 0: + await asyncio.sleep(float(request_delay_seconds)) + outcome = await process_scholar( + db_session, + pagination=pagination, + run=run, + scholar=scholar, + user_id=user_id, + start_cstart=resume_cstart, + max_pages_per_scholar=remaining_max, + auto_queue_continuations=auto_queue_continuations, + queue_delay_seconds=queue_delay_seconds, + **scholar_kwargs, + ) + apply_outcome_to_progress(progress=progress, outcome=outcome) + return progress diff --git a/app/services/scholar/source.py b/app/services/scholar/source.py index 08f22ba..8d61ec8 100644 --- a/app/services/scholar/source.py +++ b/app/services/scholar/source.py @@ -62,6 +62,7 @@ class LiveScholarSource: *, timeout_seconds: float = 25.0, min_interval_seconds: float | None = None, + rotate_user_agents: bool | None = None, user_agents: list[str] | None = None, ) -> None: self._timeout_seconds = timeout_seconds @@ -72,6 +73,38 @@ class LiveScholarSource: ) self._min_interval_seconds = max(configured_interval, 0.0) self._user_agents = user_agents or DEFAULT_USER_AGENTS + self._rotate_user_agents = ( + bool(settings.scholar_http_rotate_user_agent) + if rotate_user_agents is None + else bool(rotate_user_agents) + ) + configured_user_agent = settings.scholar_http_user_agent.strip() + self._configured_user_agent = configured_user_agent or None + self._accept_language = settings.scholar_http_accept_language.strip() or "en-US,en;q=0.9" + self._cookie_header = settings.scholar_http_cookie.strip() or None + self._stable_user_agent = self._resolve_initial_user_agent() + + def _resolve_initial_user_agent(self) -> str: + if self._configured_user_agent is not None: + return self._configured_user_agent + return random.choice(self._user_agents) + + def _resolve_user_agent_for_request(self) -> str: + if self._configured_user_agent is not None: + return self._configured_user_agent + if self._rotate_user_agents: + return random.choice(self._user_agents) + return self._stable_user_agent + + def _request_headers(self) -> dict[str, str]: + headers = { + "User-Agent": self._resolve_user_agent_for_request(), + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": self._accept_language, + } + if self._cookie_header is not None: + headers["Cookie"] = self._cookie_header + return headers async def fetch_profile_html(self, scholar_id: str) -> FetchResult: return await self.fetch_profile_page_html( @@ -116,15 +149,21 @@ class LiveScholarSource: return await asyncio.to_thread(self._fetch_sync, requested_url) def _build_request(self, requested_url: str) -> Request: - return Request( - requested_url, - headers={ - "User-Agent": random.choice(self._user_agents), - "Accept": "text/html,application/xhtml+xml", - "Accept-Language": "en-US,en;q=0.9", - "Connection": "close", - }, - ) + return Request(requested_url, headers=self._request_headers()) + + @staticmethod + def _http_error_reason(*, status_code: int, final_url: str, body: str) -> str: + lowered_url = final_url.lower() + lowered_body = body.lower() + if "sorry/index" in lowered_url or "sorry/index" in lowered_body: + return "blocked_google_sorry_challenge" + if "our systems have detected" in lowered_body or "unusual traffic" in lowered_body: + return "blocked_unusual_traffic_detected" + if "automated queries" in lowered_body: + return "blocked_automated_queries_detected" + if status_code == 429: + return "blocked_http_429_rate_limited" + return f"http_error_status_{status_code}" @staticmethod def _http_error_body(exc: HTTPError) -> str: @@ -151,18 +190,27 @@ class LiveScholarSource: @staticmethod def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult: + final_url = exc.geturl() + body = LiveScholarSource._http_error_body(exc) + block_reason = LiveScholarSource._http_error_reason( + status_code=exc.code, + final_url=final_url, + body=body, + ) structured_log( logger, "warning", "scholar_source.fetch_http_error", requested_url=requested_url, status_code=exc.code, + final_url=final_url, + block_reason=block_reason, ) return FetchResult( requested_url=requested_url, status_code=exc.code, - final_url=exc.geturl(), - body=LiveScholarSource._http_error_body(exc), + final_url=final_url, + body=body, error=str(exc), ) diff --git a/app/services/scholar/state_detection.py b/app/services/scholar/state_detection.py index 75f7125..7532afa 100644 --- a/app/services/scholar/state_detection.py +++ b/app/services/scholar/state_detection.py @@ -38,18 +38,18 @@ def classify_block_or_captcha_reason( ) -> str | None: if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url): return "blocked_accounts_redirect" - if status_code == 429: - return "blocked_http_429_rate_limited" - if status_code == 403: - if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url: - return "blocked_http_403_captcha_challenge" - return "blocked_http_403_forbidden" if "sorry/index" in final_url or "sorry/index" in body_lowered: return "blocked_google_sorry_challenge" if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered: return "blocked_unusual_traffic_detected" if "automated queries" in body_lowered: return "blocked_automated_queries_detected" + if status_code == 429: + return "blocked_http_429_rate_limited" + if status_code == 403: + if "recaptcha" in body_lowered or "captcha" in body_lowered: + return "blocked_http_403_captcha_challenge" + return "blocked_http_403_forbidden" if "not a robot" in body_lowered: return "blocked_not_a_robot_challenge" if "recaptcha" in body_lowered: diff --git a/app/settings.py b/app/settings.py index 175480e..c9098a1 100644 --- a/app/settings.py +++ b/app/settings.py @@ -235,6 +235,13 @@ class Settings: "SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD", 3, ) + scholar_http_user_agent: str = _env_str("SCHOLAR_HTTP_USER_AGENT", "") + scholar_http_rotate_user_agent: bool = _env_bool("SCHOLAR_HTTP_ROTATE_USER_AGENT", False) + scholar_http_accept_language: str = _env_str( + "SCHOLAR_HTTP_ACCEPT_LANGUAGE", + "en-US,en;q=0.9", + ) + scholar_http_cookie: str = _env_str("SCHOLAR_HTTP_COOKIE", "") unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True) unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "") unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0) diff --git a/frontend/src/features/settings/index.ts b/frontend/src/features/settings/index.ts index 6ff6c94..a1edce7 100644 --- a/frontend/src/features/settings/index.ts +++ b/frontend/src/features/settings/index.ts @@ -34,6 +34,13 @@ export interface UserSettingsUpdate { crossref_api_mailto: string | null; } +export interface AdminScholarHttpSettings { + user_agent: string; + rotate_user_agent: boolean; + accept_language: string; + cookie: string; +} + export interface ChangePasswordPayload { current_password: string; new_password: string; @@ -60,3 +67,20 @@ export async function changePassword(payload: ChangePasswordPayload): Promise<{ }); return response.data; } + +export async function fetchAdminScholarHttpSettings(): Promise { + const response = await apiRequest("/admin/settings/scholar-http", { + method: "GET", + }); + return response.data; +} + +export async function updateAdminScholarHttpSettings( + payload: AdminScholarHttpSettings, +): Promise { + const response = await apiRequest("/admin/settings/scholar-http", { + method: "PUT", + body: payload, + }); + return response.data; +} diff --git a/frontend/src/pages/SettingsPage.vue b/frontend/src/pages/SettingsPage.vue index 66f12eb..4040f0e 100644 --- a/frontend/src/pages/SettingsPage.vue +++ b/frontend/src/pages/SettingsPage.vue @@ -14,9 +14,12 @@ import AppTabs, { type AppTabItem } from "@/components/ui/AppTabs.vue"; import SettingsAdminPanel from "@/features/settings/SettingsAdminPanel.vue"; import { changePassword, + fetchAdminScholarHttpSettings, fetchSettings, + type AdminScholarHttpSettings, type UserSettings, type UserSettingsUpdate, + updateAdminScholarHttpSettings, updateSettings, } from "@/features/settings"; import { ApiRequestError } from "@/lib/api/errors"; @@ -40,6 +43,7 @@ const router = useRouter(); const loading = ref(true); const saving = ref(false); +const savingScholarHttp = ref(false); const updatingPassword = ref(false); const autoRunEnabled = ref(false); @@ -49,6 +53,10 @@ const navVisiblePages = ref([]); const openalexApiKey = ref(""); const crossrefApiToken = ref(""); const crossrefApiMailto = ref(""); +const scholarHttpUserAgent = ref(""); +const scholarHttpRotateUserAgent = ref(false); +const scholarHttpAcceptLanguage = ref("en-US,en;q=0.9"); +const scholarHttpCookie = ref(""); const currentPassword = ref(""); const newPassword = ref(""); @@ -142,6 +150,13 @@ function hydrateSettings(settings: UserSettings): void { runStatus.setSafetyState(settings.safety_state); } +function hydrateScholarHttpSettings(settings: AdminScholarHttpSettings): void { + scholarHttpUserAgent.value = settings.user_agent; + scholarHttpRotateUserAgent.value = Boolean(settings.rotate_user_agent); + scholarHttpAcceptLanguage.value = settings.accept_language; + scholarHttpCookie.value = settings.cookie; +} + function parseBoundedInteger(value: string, label: string, minimum: number): number { const parsed = Number(value); if (!Number.isInteger(parsed)) { @@ -160,6 +175,9 @@ async function loadSettings(): Promise { try { const settings = await fetchSettings(); hydrateSettings(settings); + if (auth.isAdmin) { + hydrateScholarHttpSettings(await fetchAdminScholarHttpSettings()); + } } catch (error) { if (error instanceof ApiRequestError) { errorMessage.value = error.message; @@ -172,6 +190,32 @@ async function loadSettings(): Promise { } } +async function onSaveScholarHttpSettings(): Promise { + savingScholarHttp.value = true; + errorMessage.value = null; + errorRequestId.value = null; + successMessage.value = null; + try { + const updated = await updateAdminScholarHttpSettings({ + user_agent: scholarHttpUserAgent.value.trim(), + rotate_user_agent: scholarHttpRotateUserAgent.value, + accept_language: scholarHttpAcceptLanguage.value.trim(), + cookie: scholarHttpCookie.value.trim(), + }); + hydrateScholarHttpSettings(updated); + successMessage.value = "Scholar HTTP profile updated."; + } catch (error) { + if (error instanceof ApiRequestError) { + errorMessage.value = error.message; + errorRequestId.value = error.requestId; + } else { + errorMessage.value = "Unable to save Scholar HTTP profile."; + } + } finally { + savingScholarHttp.value = false; + } +} + async function onSaveSettings(): Promise { saving.value = true; errorMessage.value = null; @@ -404,6 +448,49 @@ onMounted(async () => { + +
+

Scholar Request Profile

+

+ Tune Scholar HTTP fingerprint behavior used by live scraper requests. Changes apply to new runs. +

+ + + +
+ +
+
+ + {{ savingScholarHttp ? "Saving..." : "Save Scholar request profile" }} + +
+
diff --git a/tests/integration/test_api_v1.py b/tests/integration/test_api_v1.py index 41a9051..06551ca 100644 --- a/tests/integration/test_api_v1.py +++ b/tests/integration/test_api_v1.py @@ -391,6 +391,61 @@ async def test_api_admin_user_management_endpoints(db_session: AsyncSession) -> assert created_exists.scalar_one() == 1 +@pytest.mark.integration +@pytest.mark.db +@pytest.mark.asyncio +async def test_api_admin_scholar_http_settings_endpoints(db_session: AsyncSession) -> None: + await insert_user( + db_session, + email="api-admin-http@example.com", + password="admin-password", + is_admin=True, + ) + await insert_user( + db_session, + email="api-member-http@example.com", + password="member-password", + is_admin=False, + ) + previous_user_agent = settings.scholar_http_user_agent + previous_rotate = settings.scholar_http_rotate_user_agent + previous_accept_language = settings.scholar_http_accept_language + previous_cookie = settings.scholar_http_cookie + try: + client = TestClient(app) + login_user(client, email="api-admin-http@example.com", password="admin-password") + headers = _api_csrf_headers(client) + + read_response = client.get("/api/v1/admin/settings/scholar-http") + assert read_response.status_code == 200 + + update_response = client.put( + "/api/v1/admin/settings/scholar-http", + json={ + "user_agent": "Mozilla/5.0 Test Runner", + "rotate_user_agent": False, + "accept_language": "en-US,en;q=0.8", + "cookie": "SID=test-cookie", + }, + headers=headers, + ) + assert update_response.status_code == 200 + payload = update_response.json()["data"] + assert payload["user_agent"] == "Mozilla/5.0 Test Runner" + assert payload["cookie"] == "SID=test-cookie" + assert settings.scholar_http_user_agent == "Mozilla/5.0 Test Runner" + + client.post("/api/v1/auth/logout", headers=headers) + login_user(client, email="api-member-http@example.com", password="member-password") + forbidden_response = client.get("/api/v1/admin/settings/scholar-http") + assert forbidden_response.status_code == 403 + finally: + object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent) + object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate) + object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language) + object.__setattr__(settings, "scholar_http_cookie", previous_cookie) + + @pytest.mark.integration @pytest.mark.db @pytest.mark.asyncio diff --git a/tests/integration/test_deferred_enrichment.py b/tests/integration/test_deferred_enrichment.py index 14f5bee..d026fe3 100644 --- a/tests/integration/test_deferred_enrichment.py +++ b/tests/integration/test_deferred_enrichment.py @@ -8,7 +8,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType, ScholarProfile, ScholarPublication -from app.services.ingestion.application import ScholarIngestionService +from app.services.ingestion.enrichment import EnrichmentRunner from app.services.openalex.types import OpenAlexWork from tests.integration.helpers import insert_user @@ -79,8 +79,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession oa_url="http://example.com/grover.pdf", ) - mock_source = MagicMock() - service = ScholarIngestionService(source=mock_source) + runner = EnrichmentRunner() # We patch the client at its source, and also mock arXiv to avoid real HTTP calls with ( @@ -93,7 +92,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work]) # 5. Execute the enrichment pass for the NEW run - await service._enrich_pending_publications(db_session, run_id=new_run.id) + await runner.enrich_pending_publications(db_session, run_id=new_run.id) await db_session.commit() # 6. Verification: The publication from the FAILED run should now be enriched diff --git a/tests/unit/test_ingestion_arxiv_rate_limit.py b/tests/unit/test_ingestion_arxiv_rate_limit.py index 68f2792..5d38779 100644 --- a/tests/unit/test_ingestion_arxiv_rate_limit.py +++ b/tests/unit/test_ingestion_arxiv_rate_limit.py @@ -5,7 +5,7 @@ from types import SimpleNamespace import pytest from app.services.arxiv.errors import ArxivRateLimitError -from app.services.ingestion.application import ScholarIngestionService +from app.services.ingestion.enrichment import EnrichmentRunner from app.services.publication_identifiers import application as identifier_service @@ -13,7 +13,7 @@ from app.services.publication_identifiers import application as identifier_servi async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit( monkeypatch: pytest.MonkeyPatch, ) -> None: - service = ScholarIngestionService(source=object()) + runner = EnrichmentRunner() publication = SimpleNamespace(id=11, author_text="Ada Lovelace") calls = {"sync": 0} @@ -35,9 +35,9 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit( async def _publish_noop(*args, **kwargs) -> None: _ = (args, kwargs) - monkeypatch.setattr(service, "_publish_identifier_update_event", _publish_noop) + monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop) - result = await service._discover_identifiers_for_enrichment( + result = await runner._discover_identifiers_for_enrichment( object(), publication=publication, run_id=321, diff --git a/tests/unit/test_ingestion_progress_reporting.py b/tests/unit/test_ingestion_progress_reporting.py index ed470d6..bdae206 100644 --- a/tests/unit/test_ingestion_progress_reporting.py +++ b/tests/unit/test_ingestion_progress_reporting.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -from app.services.ingestion.application import ScholarIngestionService +from app.services.ingestion.run_completion import apply_outcome_to_progress from app.services.ingestion.types import RunProgress, ScholarProcessingOutcome @@ -31,11 +31,11 @@ def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessin def test_apply_outcome_to_progress_replaces_previous_scholar_outcome() -> None: progress = RunProgress() - ScholarIngestionService._apply_outcome_to_progress( + apply_outcome_to_progress( progress=progress, outcome=_outcome(scholar_profile_id=42, outcome_label="partial"), ) - ScholarIngestionService._apply_outcome_to_progress( + apply_outcome_to_progress( progress=progress, outcome=_outcome(scholar_profile_id=42, outcome_label="success"), ) @@ -58,4 +58,4 @@ def test_apply_outcome_to_progress_rejects_invalid_scholar_id() -> None: ) with pytest.raises(RuntimeError, match="missing valid scholar_profile_id"): - ScholarIngestionService._apply_outcome_to_progress(progress=progress, outcome=invalid) + apply_outcome_to_progress(progress=progress, outcome=invalid) diff --git a/tests/unit/test_scholar_parser.py b/tests/unit/test_scholar_parser.py index ee3c3ec..90bc2da 100644 --- a/tests/unit/test_scholar_parser.py +++ b/tests/unit/test_scholar_parser.py @@ -433,3 +433,18 @@ def test_parse_author_search_page_classifies_http_429_as_blocked() -> None: assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA assert parsed.state_reason == "blocked_http_429_rate_limited" + + +def test_parse_author_search_page_prefers_sorry_challenge_reason_over_generic_429() -> None: + fetch_result = FetchResult( + requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada", + status_code=429, + final_url="https://www.google.com/sorry/index?continue=scholar", + body="Our systems have detected unusual traffic.", + error=None, + ) + + parsed = parse_author_search_page(fetch_result) + + assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA + assert parsed.state_reason == "blocked_google_sorry_challenge" diff --git a/tests/unit/test_scholar_source.py b/tests/unit/test_scholar_source.py index 654cb58..0d98f4c 100644 --- a/tests/unit/test_scholar_source.py +++ b/tests/unit/test_scholar_source.py @@ -2,6 +2,12 @@ import pytest from app.services.scholar import rate_limit as scholar_rate_limit from app.services.scholar.source import FetchResult, LiveScholarSource, _build_profile_url +from app.settings import settings + + +def _request_header(request, name: str) -> str | None: + headers = {key.lower(): value for key, value in request.header_items()} + return headers.get(name.lower()) def test_build_profile_url_includes_pagesize_for_initial_page() -> None: @@ -48,3 +54,72 @@ async def test_live_scholar_source_applies_global_throttle(monkeypatch: pytest.M assert result == expected_result assert captured_interval == [7.0] + + +def test_http_error_reason_prefers_sorry_challenge_marker() -> None: + reason = LiveScholarSource._http_error_reason( + status_code=429, + final_url="https://www.google.com/sorry/index?continue=example", + body="", + ) + assert reason == "blocked_google_sorry_challenge" + + +def test_http_error_reason_falls_back_to_http_429_rate_limit() -> None: + reason = LiveScholarSource._http_error_reason( + status_code=429, + final_url="https://scholar.google.com/citations?hl=en&user=abc", + body="Too Many Requests", + ) + assert reason == "blocked_http_429_rate_limited" + + +def test_build_request_uses_stable_user_agent_by_default() -> None: + previous_user_agent = settings.scholar_http_user_agent + previous_rotate = settings.scholar_http_rotate_user_agent + previous_cookie = settings.scholar_http_cookie + previous_accept_language = settings.scholar_http_accept_language + object.__setattr__(settings, "scholar_http_user_agent", "") + object.__setattr__(settings, "scholar_http_rotate_user_agent", False) + object.__setattr__(settings, "scholar_http_cookie", "") + object.__setattr__(settings, "scholar_http_accept_language", "en-US,en;q=0.9") + try: + source = LiveScholarSource(user_agents=["UA-A", "UA-B"]) + request_one = source._build_request("https://example.test/one") + request_two = source._build_request("https://example.test/two") + assert _request_header(request_one, "User-Agent") == _request_header(request_two, "User-Agent") + assert _request_header(request_one, "Connection") is None + finally: + object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent) + object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate) + object.__setattr__(settings, "scholar_http_cookie", previous_cookie) + object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language) + + +def test_build_request_rotates_user_agent_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + previous_user_agent = settings.scholar_http_user_agent + previous_rotate = settings.scholar_http_rotate_user_agent + object.__setattr__(settings, "scholar_http_user_agent", "") + object.__setattr__(settings, "scholar_http_rotate_user_agent", True) + choices = iter(["UA-STABLE", "UA-1", "UA-2"]) + monkeypatch.setattr("app.services.scholar.source.random.choice", lambda _values: next(choices)) + try: + source = LiveScholarSource(user_agents=["UA-A", "UA-B"]) + request_one = source._build_request("https://example.test/one") + request_two = source._build_request("https://example.test/two") + assert _request_header(request_one, "User-Agent") == "UA-1" + assert _request_header(request_two, "User-Agent") == "UA-2" + finally: + object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent) + object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate) + + +def test_build_request_includes_cookie_when_configured() -> None: + previous_cookie = settings.scholar_http_cookie + object.__setattr__(settings, "scholar_http_cookie", "SID=abc123") + try: + source = LiveScholarSource(user_agents=["UA-A"]) + request = source._build_request("https://example.test/one") + assert _request_header(request, "Cookie") == "SID=abc123" + finally: + object.__setattr__(settings, "scholar_http_cookie", previous_cookie)