decompose application.py into enrichment, scholar processing, run completion
This commit is contained in:
parent
9a7fa5004e
commit
6379c97e90
28 changed files with 2982 additions and 2576 deletions
|
|
@ -115,6 +115,10 @@ SCHOLAR_NAME_SEARCH_COOLDOWN_BLOCK_THRESHOLD=1
|
|||
SCHOLAR_NAME_SEARCH_COOLDOWN_SECONDS=1800
|
||||
SCHOLAR_NAME_SEARCH_ALERT_RETRY_COUNT_THRESHOLD=2
|
||||
SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD=3
|
||||
SCHOLAR_HTTP_USER_AGENT=
|
||||
SCHOLAR_HTTP_ROTATE_USER_AGENT=0
|
||||
SCHOLAR_HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.9
|
||||
SCHOLAR_HTTP_COOKIE=
|
||||
|
||||
# ------------------------------
|
||||
# OA Enrichment + PDF Resolution
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ This file contains self-contained decomposition prompts ("slices") to bring the
|
|||
|---|-------|--------|
|
||||
| 1 | Schema split | **DONE** |
|
||||
| 2 | Ingestion: pagination + publication upsert | **DONE** |
|
||||
| 3 | Ingestion: enrichment + scholar processing + run completion | pending |
|
||||
| 3 | Ingestion: enrichment + scholar processing + run completion | **DONE** |
|
||||
| 4 | Scholars service + PDF queue | pending |
|
||||
| 5 | Routers + scheduler | pending |
|
||||
|
||||
|
|
@ -63,7 +63,17 @@ Also fixed two external import paths (`portability/normalize.py`, `portability/p
|
|||
|
||||
---
|
||||
|
||||
## Slice 3: Decompose `app/services/ingestion/application.py` — enrichment, scholar processing, run completion
|
||||
## Slice 3: Decompose `app/services/ingestion/application.py` — enrichment, scholar processing, run completion — DONE
|
||||
|
||||
Completed. Extracted four logical chunks from `application.py` (2,085 → 635 lines):
|
||||
|
||||
| File | Lines | Contents |
|
||||
|---|---|---|
|
||||
| `app/services/ingestion/enrichment.py` | 323 | `EnrichmentRunner` class — OpenAlex enrichment, identifier discovery, dedup sweep |
|
||||
| `app/services/ingestion/scholar_processing.py` | 614 | `process_scholar`, `run_scholar_iteration`, continuation queue, outcome resolution |
|
||||
| `app/services/ingestion/run_completion.py` | 417 | `complete_run_for_user`, failure summary, alert summary, safety outcome, progress tracking |
|
||||
|
||||
Also updated two tests (`test_ingestion_arxiv_rate_limit.py`, `test_ingestion_progress_reporting.py`) and one integration test (`test_deferred_enrichment.py`) to import from new modules.
|
||||
|
||||
**Why:** Three remaining logical chunks. Extracting all three brings `application.py` down to ~500 lines (the orchestration spine).
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@ from __future__ import annotations
|
|||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.routers import admin, admin_dbops, auth, publications, runs, scholars, settings
|
||||
from app.api.routers import admin, admin_dbops, admin_settings, auth, publications, runs, scholars, settings
|
||||
|
||||
router = APIRouter(prefix="/api/v1")
|
||||
router.include_router(auth.router)
|
||||
router.include_router(admin.router)
|
||||
router.include_router(admin_settings.router)
|
||||
router.include_router(admin_dbops.router)
|
||||
router.include_router(scholars.router)
|
||||
router.include_router(settings.router)
|
||||
|
|
|
|||
104
app/api/routers/admin_settings.py
Normal file
104
app/api/routers/admin_settings.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from app.api.deps import get_api_admin_user
|
||||
from app.api.errors import ApiException
|
||||
from app.api.responses import success_payload
|
||||
from app.api.schemas import (
|
||||
AdminScholarHttpSettingsEnvelope,
|
||||
AdminScholarHttpSettingsUpdateRequest,
|
||||
)
|
||||
from app.db.models import User
|
||||
from app.logging_utils import structured_log
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/admin/settings", tags=["api-admin-settings"])
|
||||
_MAX_USER_AGENT_LENGTH = 512
|
||||
_MAX_ACCEPT_LANGUAGE_LENGTH = 128
|
||||
_MAX_COOKIE_LENGTH = 8192
|
||||
|
||||
|
||||
def _normalize_header_value(value: str, *, field_name: str, max_length: int) -> str:
|
||||
normalized = value.strip()
|
||||
if len(normalized) > max_length:
|
||||
raise ApiException(
|
||||
status_code=400,
|
||||
code="invalid_admin_setting",
|
||||
message=f"{field_name} must be {max_length} characters or fewer.",
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _serialize_scholar_http_settings() -> dict[str, object]:
|
||||
return {
|
||||
"user_agent": settings.scholar_http_user_agent,
|
||||
"rotate_user_agent": bool(settings.scholar_http_rotate_user_agent),
|
||||
"accept_language": settings.scholar_http_accept_language,
|
||||
"cookie": settings.scholar_http_cookie,
|
||||
}
|
||||
|
||||
|
||||
def _apply_scholar_http_settings(payload: AdminScholarHttpSettingsUpdateRequest) -> None:
|
||||
user_agent = _normalize_header_value(
|
||||
payload.user_agent,
|
||||
field_name="user_agent",
|
||||
max_length=_MAX_USER_AGENT_LENGTH,
|
||||
)
|
||||
accept_language = _normalize_header_value(
|
||||
payload.accept_language,
|
||||
field_name="accept_language",
|
||||
max_length=_MAX_ACCEPT_LANGUAGE_LENGTH,
|
||||
)
|
||||
cookie = _normalize_header_value(
|
||||
payload.cookie,
|
||||
field_name="cookie",
|
||||
max_length=_MAX_COOKIE_LENGTH,
|
||||
)
|
||||
object.__setattr__(settings, "scholar_http_user_agent", user_agent)
|
||||
object.__setattr__(settings, "scholar_http_rotate_user_agent", bool(payload.rotate_user_agent))
|
||||
object.__setattr__(settings, "scholar_http_accept_language", accept_language)
|
||||
object.__setattr__(settings, "scholar_http_cookie", cookie)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/scholar-http",
|
||||
response_model=AdminScholarHttpSettingsEnvelope,
|
||||
)
|
||||
async def get_scholar_http_settings(
|
||||
request: Request,
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"api.admin.settings.scholar_http_read",
|
||||
admin_user_id=int(admin_user.id),
|
||||
)
|
||||
return success_payload(request, data=_serialize_scholar_http_settings())
|
||||
|
||||
|
||||
@router.put(
|
||||
"/scholar-http",
|
||||
response_model=AdminScholarHttpSettingsEnvelope,
|
||||
)
|
||||
async def update_scholar_http_settings(
|
||||
payload: AdminScholarHttpSettingsUpdateRequest,
|
||||
request: Request,
|
||||
admin_user: User = Depends(get_api_admin_user),
|
||||
):
|
||||
_apply_scholar_http_settings(payload)
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"api.admin.settings.scholar_http_updated",
|
||||
admin_user_id=int(admin_user.id),
|
||||
rotate_user_agent=bool(settings.scholar_http_rotate_user_agent),
|
||||
user_agent_set=bool(settings.scholar_http_user_agent.strip()),
|
||||
cookie_set=bool(settings.scholar_http_cookie.strip()),
|
||||
)
|
||||
return success_payload(request, data=_serialize_scholar_http_settings())
|
||||
|
|
@ -1,945 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
|
||||
class ApiMeta(BaseModel):
|
||||
request_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ApiErrorData(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
details: Any | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ApiErrorEnvelope(BaseModel):
|
||||
error: ApiErrorData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MessageData(BaseModel):
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MessageEnvelope(BaseModel):
|
||||
data: MessageData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SessionUserData(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
is_admin: bool
|
||||
is_active: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AuthMeData(BaseModel):
|
||||
authenticated: bool
|
||||
csrf_token: str
|
||||
user: SessionUserData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AuthMeEnvelope(BaseModel):
|
||||
data: AuthMeData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CsrfBootstrapData(BaseModel):
|
||||
csrf_token: str
|
||||
authenticated: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CsrfBootstrapEnvelope(BaseModel):
|
||||
data: CsrfBootstrapData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginData(BaseModel):
|
||||
authenticated: bool
|
||||
csrf_token: str
|
||||
user: SessionUserData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginEnvelope(BaseModel):
|
||||
data: LoginData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
current_password: str
|
||||
new_password: str
|
||||
confirm_password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarItemData(BaseModel):
|
||||
id: int
|
||||
scholar_id: str
|
||||
display_name: str | None
|
||||
profile_image_url: str | None
|
||||
profile_image_source: str
|
||||
is_enabled: bool
|
||||
baseline_completed: bool
|
||||
last_run_dt: datetime | None
|
||||
last_run_status: str | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarsListData(BaseModel):
|
||||
scholars: list[ScholarItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarsListEnvelope(BaseModel):
|
||||
data: ScholarsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarCreateRequest(BaseModel):
|
||||
scholar_id: str
|
||||
profile_image_url: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarEnvelope(BaseModel):
|
||||
data: ScholarItemData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchCandidateData(BaseModel):
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str] = Field(default_factory=list)
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchData(BaseModel):
|
||||
query: str
|
||||
state: str
|
||||
state_reason: str
|
||||
action_hint: str | None = None
|
||||
candidates: list[ScholarSearchCandidateData]
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchEnvelope(BaseModel):
|
||||
data: ScholarSearchData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarImageUrlUpdateRequest(BaseModel):
|
||||
image_url: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarExportItemData(BaseModel):
|
||||
scholar_id: str
|
||||
display_name: str | None = None
|
||||
is_enabled: bool = True
|
||||
profile_image_override_url: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationExportItemData(BaseModel):
|
||||
scholar_id: str
|
||||
cluster_id: str | None = None
|
||||
fingerprint_sha256: str | None = None
|
||||
title: str
|
||||
year: int | None = None
|
||||
citation_count: int = 0
|
||||
author_text: str | None = None
|
||||
venue_text: str | None = None
|
||||
pub_url: str | None = None
|
||||
pdf_url: str | None = None
|
||||
is_read: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataExportData(BaseModel):
|
||||
schema_version: int
|
||||
exported_at: str
|
||||
scholars: list[ScholarExportItemData]
|
||||
publications: list[PublicationExportItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataExportEnvelope(BaseModel):
|
||||
data: DataExportData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportRequest(BaseModel):
|
||||
schema_version: int | None = None
|
||||
scholars: list[ScholarExportItemData] = Field(default_factory=list)
|
||||
publications: list[PublicationExportItemData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportResultData(BaseModel):
|
||||
scholars_created: int
|
||||
scholars_updated: int
|
||||
publications_created: int
|
||||
publications_updated: int
|
||||
links_created: int
|
||||
links_updated: int
|
||||
skipped_records: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportEnvelope(BaseModel):
|
||||
data: DataImportResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunListItemData(BaseModel):
|
||||
id: int
|
||||
trigger_type: str
|
||||
status: str
|
||||
start_dt: datetime
|
||||
end_dt: datetime | None
|
||||
scholar_count: int
|
||||
new_publication_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunsListData(BaseModel):
|
||||
runs: list[RunListItemData]
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunsListEnvelope(BaseModel):
|
||||
data: RunsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunSummaryData(BaseModel):
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
failed_state_counts: dict[str, int] = Field(default_factory=dict)
|
||||
failed_reason_counts: dict[str, int] = Field(default_factory=dict)
|
||||
scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
|
||||
retry_counts: dict[str, int] = Field(default_factory=dict)
|
||||
alert_thresholds: dict[str, int] = Field(default_factory=dict)
|
||||
alert_flags: dict[str, bool] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScrapeSafetyCountersData(BaseModel):
|
||||
consecutive_blocked_runs: int = 0
|
||||
consecutive_network_runs: int = 0
|
||||
cooldown_entry_count: int = 0
|
||||
blocked_start_count: int = 0
|
||||
last_blocked_failure_count: int = 0
|
||||
last_network_failure_count: int = 0
|
||||
last_evaluated_run_id: int | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScrapeSafetyStateData(BaseModel):
|
||||
cooldown_active: bool
|
||||
cooldown_reason: str | None = None
|
||||
cooldown_reason_label: str | None = None
|
||||
cooldown_until: datetime | None = None
|
||||
cooldown_remaining_seconds: int = 0
|
||||
recommended_action: str | None = None
|
||||
counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunAttemptLogData(BaseModel):
|
||||
attempt: int
|
||||
cstart: int
|
||||
state: str | None = None
|
||||
state_reason: str | None = None
|
||||
status_code: int | None = None
|
||||
fetch_error: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunPageLogData(BaseModel):
|
||||
page: int
|
||||
cstart: int
|
||||
state: str
|
||||
state_reason: str | None = None
|
||||
status_code: int | None = None
|
||||
publication_count: int = 0
|
||||
attempt_count: int = 0
|
||||
has_show_more_button: bool = False
|
||||
articles_range: str | None = None
|
||||
warning_codes: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDebugData(BaseModel):
|
||||
status_code: int | None = None
|
||||
final_url: str | None = None
|
||||
fetch_error: str | None = None
|
||||
body_sha256: str | None = None
|
||||
body_length: int | None = None
|
||||
has_show_more_button: bool | None = None
|
||||
articles_range: str | None = None
|
||||
state_reason: str | None = None
|
||||
warning_codes: list[str] = Field(default_factory=list)
|
||||
marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
|
||||
page_logs: list[RunPageLogData] = Field(default_factory=list)
|
||||
attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunScholarResultData(BaseModel):
|
||||
scholar_profile_id: int
|
||||
scholar_id: str
|
||||
state: str
|
||||
state_reason: str | None = None
|
||||
outcome: str
|
||||
attempt_count: int = 0
|
||||
publication_count: int = 0
|
||||
start_cstart: int = 0
|
||||
continuation_cstart: int | None = None
|
||||
continuation_enqueued: bool = False
|
||||
continuation_cleared: bool = False
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
error: str | None = None
|
||||
debug: RunDebugData | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDetailData(BaseModel):
|
||||
run: RunListItemData
|
||||
summary: RunSummaryData
|
||||
scholar_results: list[RunScholarResultData]
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDetailEnvelope(BaseModel):
|
||||
data: RunDetailData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ManualRunData(BaseModel):
|
||||
run_id: int
|
||||
status: str
|
||||
scholar_count: int
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
new_publication_count: int
|
||||
reused_existing_run: bool
|
||||
idempotency_key: str | None = None
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ManualRunEnvelope(BaseModel):
|
||||
data: ManualRunData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueItemData(BaseModel):
|
||||
id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
status: str
|
||||
reason: str
|
||||
dropped_reason: str | None
|
||||
attempt_count: int
|
||||
resume_cstart: int
|
||||
next_attempt_dt: datetime | None
|
||||
updated_at: datetime
|
||||
last_error: str | None
|
||||
last_run_id: int | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueListData(BaseModel):
|
||||
queue_items: list[QueueItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueListEnvelope(BaseModel):
|
||||
data: QueueListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueItemEnvelope(BaseModel):
|
||||
data: QueueItemData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueClearData(BaseModel):
|
||||
queue_item_id: int
|
||||
previous_status: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueClearEnvelope(BaseModel):
|
||||
data: QueueClearData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserData(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
is_active: bool
|
||||
is_admin: bool
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUsersListData(BaseModel):
|
||||
users: list[AdminUserData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUsersListEnvelope(BaseModel):
|
||||
data: AdminUsersListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserCreateRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
is_admin: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserActiveUpdateRequest(BaseModel):
|
||||
is_active: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserEnvelope(BaseModel):
|
||||
data: AdminUserData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminResetPasswordRequest(BaseModel):
|
||||
new_password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityCheckData(BaseModel):
|
||||
name: str
|
||||
count: int
|
||||
severity: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityData(BaseModel):
|
||||
status: str
|
||||
checked_at: datetime
|
||||
failures: list[str] = Field(default_factory=list)
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityEnvelope(BaseModel):
|
||||
data: AdminDbIntegrityData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobData(BaseModel):
|
||||
id: int
|
||||
job_name: str
|
||||
requested_by: str | None
|
||||
dry_run: bool
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
error_text: str | None
|
||||
started_at: datetime | None
|
||||
finished_at: datetime | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobsData(BaseModel):
|
||||
jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobsEnvelope(BaseModel):
|
||||
data: AdminDbRepairJobsData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DisplayIdentifierData(BaseModel):
|
||||
kind: str
|
||||
value: str
|
||||
label: str
|
||||
url: str | None
|
||||
confidence_score: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueItemData(BaseModel):
|
||||
publication_id: int
|
||||
title: str
|
||||
display_identifier: DisplayIdentifierData | None = None
|
||||
pdf_url: str | None
|
||||
status: str
|
||||
attempt_count: int
|
||||
last_failure_reason: str | None
|
||||
last_failure_detail: str | None
|
||||
last_source: str | None
|
||||
requested_by_user_id: int | None
|
||||
requested_by_email: str | None
|
||||
queued_at: datetime | None
|
||||
last_attempt_at: datetime | None
|
||||
resolved_at: datetime | None
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueData(BaseModel):
|
||||
items: list[AdminPdfQueueItemData] = Field(default_factory=list)
|
||||
total_count: int = 0
|
||||
page: int = 1
|
||||
page_size: int = 100
|
||||
has_next: bool = False
|
||||
has_prev: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueRequeueData(BaseModel):
|
||||
publication_id: int
|
||||
queued: bool
|
||||
status: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueRequeueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueRequeueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueBulkEnqueueData(BaseModel):
|
||||
requested_count: int
|
||||
queued_count: int
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueBulkEnqueueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksRequest(BaseModel):
|
||||
scope_mode: Literal["single_user", "all_users"] = "single_user"
|
||||
user_id: int | None = Field(default=None, ge=1)
|
||||
scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
|
||||
dry_run: bool = True
|
||||
gc_orphan_publications: bool = False
|
||||
requested_by: str | None = None
|
||||
confirmation_text: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_scope(self) -> AdminRepairPublicationLinksRequest:
|
||||
if self.scope_mode == "single_user" and self.user_id is None:
|
||||
raise ValueError("user_id is required when scope_mode=single_user.")
|
||||
if self.scope_mode == "all_users" and self.user_id is not None:
|
||||
raise ValueError("user_id must be omitted when scope_mode=all_users.")
|
||||
if not self.dry_run and self.scope_mode == "all_users":
|
||||
expected = "REPAIR ALL USERS"
|
||||
provided = (self.confirmation_text or "").strip()
|
||||
if provided != expected:
|
||||
raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
|
||||
return self
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksResultData(BaseModel):
|
||||
job_id: int
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksEnvelope(BaseModel):
|
||||
data: AdminRepairPublicationLinksResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminNearDuplicateClusterMemberData(BaseModel):
|
||||
publication_id: int
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminNearDuplicateClusterData(BaseModel):
|
||||
cluster_key: str
|
||||
winner_publication_id: int
|
||||
member_count: int
|
||||
similarity_score: float = Field(ge=0.0, le=1.0)
|
||||
members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
|
||||
dry_run: bool = True
|
||||
similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
|
||||
min_shared_tokens: int = Field(default=3, ge=1, le=8)
|
||||
max_year_delta: int = Field(default=1, ge=0, le=5)
|
||||
max_clusters: int = Field(default=25, ge=1, le=200)
|
||||
selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
|
||||
requested_by: str | None = None
|
||||
confirmation_text: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
|
||||
if self.dry_run:
|
||||
return self
|
||||
if not self.selected_cluster_keys:
|
||||
raise ValueError("selected_cluster_keys is required when dry_run=false.")
|
||||
expected = "MERGE SELECTED DUPLICATES"
|
||||
provided = (self.confirmation_text or "").strip()
|
||||
if provided != expected:
|
||||
raise ValueError(
|
||||
"confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
|
||||
job_id: int
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
|
||||
data: AdminRepairPublicationNearDuplicatesResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsPolicyData(BaseModel):
|
||||
min_run_interval_minutes: int
|
||||
min_request_delay_seconds: int
|
||||
automation_allowed: bool
|
||||
manual_run_allowed: bool
|
||||
blocked_failure_threshold: int
|
||||
network_failure_threshold: int
|
||||
cooldown_blocked_seconds: int
|
||||
cooldown_network_seconds: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsData(BaseModel):
|
||||
auto_run_enabled: bool
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
nav_visible_pages: list[str]
|
||||
policy: SettingsPolicyData
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
openalex_api_key: str | None = None
|
||||
crossref_api_token: str | None = None
|
||||
crossref_api_mailto: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsEnvelope(BaseModel):
|
||||
data: SettingsData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsUpdateRequest(BaseModel):
|
||||
auto_run_enabled: bool
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
nav_visible_pages: list[str] | None = None
|
||||
|
||||
openalex_api_key: str | None = None
|
||||
crossref_api_token: str | None = None
|
||||
crossref_api_mailto: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationItemData(BaseModel):
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
display_identifier: DisplayIdentifierData | None = None
|
||||
pdf_url: str | None
|
||||
pdf_status: str = "untracked"
|
||||
pdf_attempt_count: int = 0
|
||||
pdf_failure_reason: str | None = None
|
||||
pdf_failure_detail: str | None = None
|
||||
is_read: bool
|
||||
is_favorite: bool = False
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationsListData(BaseModel):
|
||||
mode: str
|
||||
favorite_only: bool = False
|
||||
selected_scholar_profile_id: int | None
|
||||
unread_count: int
|
||||
favorites_count: int
|
||||
latest_count: int
|
||||
new_count: int
|
||||
total_count: int
|
||||
page: int
|
||||
page_size: int
|
||||
snapshot: str
|
||||
has_next: bool = False
|
||||
has_prev: bool = False
|
||||
publications: list[PublicationItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationsListEnvelope(BaseModel):
|
||||
data: PublicationsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkAllReadData(BaseModel):
|
||||
message: str
|
||||
updated_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkAllReadEnvelope(BaseModel):
|
||||
data: MarkAllReadData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationSelectionItem(BaseModel):
|
||||
scholar_profile_id: int
|
||||
publication_id: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadRequest(BaseModel):
|
||||
selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadData(BaseModel):
|
||||
message: str
|
||||
requested_count: int
|
||||
updated_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadEnvelope(BaseModel):
|
||||
data: MarkSelectedReadData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfRequest(BaseModel):
|
||||
scholar_profile_id: int = Field(ge=1)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfData(BaseModel):
|
||||
message: str
|
||||
queued: bool
|
||||
resolved_pdf: bool
|
||||
publication: PublicationItemData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfEnvelope(BaseModel):
|
||||
data: RetryPublicationPdfData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteRequest(BaseModel):
|
||||
scholar_profile_id: int = Field(ge=1)
|
||||
is_favorite: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteData(BaseModel):
|
||||
message: str
|
||||
publication: PublicationItemData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteEnvelope(BaseModel):
|
||||
data: TogglePublicationFavoriteData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
7
app/api/schemas/__init__.py
Normal file
7
app/api/schemas/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
from app.api.schemas.admin import * # noqa: F401, F403
|
||||
from app.api.schemas.auth import * # noqa: F401, F403
|
||||
from app.api.schemas.common import * # noqa: F401, F403
|
||||
from app.api.schemas.publications import * # noqa: F401, F403
|
||||
from app.api.schemas.runs import * # noqa: F401, F403
|
||||
from app.api.schemas.scholars import * # noqa: F401, F403
|
||||
from app.api.schemas.settings import * # noqa: F401, F403
|
||||
314
app/api/schemas/admin.py
Normal file
314
app/api/schemas/admin.py
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from app.api.schemas.common import ApiMeta
|
||||
from app.api.schemas.publications import DisplayIdentifierData
|
||||
|
||||
|
||||
class AdminUserData(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
is_active: bool
|
||||
is_admin: bool
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUsersListData(BaseModel):
|
||||
users: list[AdminUserData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUsersListEnvelope(BaseModel):
|
||||
data: AdminUsersListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserCreateRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
is_admin: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserActiveUpdateRequest(BaseModel):
|
||||
is_active: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminUserEnvelope(BaseModel):
|
||||
data: AdminUserData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminResetPasswordRequest(BaseModel):
|
||||
new_password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminScholarHttpSettingsData(BaseModel):
|
||||
user_agent: str
|
||||
rotate_user_agent: bool
|
||||
accept_language: str
|
||||
cookie: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminScholarHttpSettingsEnvelope(BaseModel):
|
||||
data: AdminScholarHttpSettingsData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminScholarHttpSettingsUpdateRequest(BaseModel):
|
||||
user_agent: str
|
||||
rotate_user_agent: bool
|
||||
accept_language: str
|
||||
cookie: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityCheckData(BaseModel):
|
||||
name: str
|
||||
count: int
|
||||
severity: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityData(BaseModel):
|
||||
status: str
|
||||
checked_at: datetime
|
||||
failures: list[str] = Field(default_factory=list)
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
checks: list[AdminDbIntegrityCheckData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbIntegrityEnvelope(BaseModel):
|
||||
data: AdminDbIntegrityData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobData(BaseModel):
|
||||
id: int
|
||||
job_name: str
|
||||
requested_by: str | None
|
||||
dry_run: bool
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
error_text: str | None
|
||||
started_at: datetime | None
|
||||
finished_at: datetime | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobsData(BaseModel):
|
||||
jobs: list[AdminDbRepairJobData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminDbRepairJobsEnvelope(BaseModel):
|
||||
data: AdminDbRepairJobsData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueItemData(BaseModel):
|
||||
publication_id: int
|
||||
title: str
|
||||
display_identifier: DisplayIdentifierData | None = None
|
||||
pdf_url: str | None
|
||||
status: str
|
||||
attempt_count: int
|
||||
last_failure_reason: str | None
|
||||
last_failure_detail: str | None
|
||||
last_source: str | None
|
||||
requested_by_user_id: int | None
|
||||
requested_by_email: str | None
|
||||
queued_at: datetime | None
|
||||
last_attempt_at: datetime | None
|
||||
resolved_at: datetime | None
|
||||
updated_at: datetime
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueData(BaseModel):
|
||||
items: list[AdminPdfQueueItemData] = Field(default_factory=list)
|
||||
total_count: int = 0
|
||||
page: int = 1
|
||||
page_size: int = 100
|
||||
has_next: bool = False
|
||||
has_prev: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueRequeueData(BaseModel):
|
||||
publication_id: int
|
||||
queued: bool
|
||||
status: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueRequeueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueRequeueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueBulkEnqueueData(BaseModel):
|
||||
requested_count: int
|
||||
queued_count: int
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminPdfQueueBulkEnqueueEnvelope(BaseModel):
|
||||
data: AdminPdfQueueBulkEnqueueData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksRequest(BaseModel):
|
||||
scope_mode: Literal["single_user", "all_users"] = "single_user"
|
||||
user_id: int | None = Field(default=None, ge=1)
|
||||
scholar_profile_ids: list[int] = Field(default_factory=list, max_length=200)
|
||||
dry_run: bool = True
|
||||
gc_orphan_publications: bool = False
|
||||
requested_by: str | None = None
|
||||
confirmation_text: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_scope(self) -> AdminRepairPublicationLinksRequest:
|
||||
if self.scope_mode == "single_user" and self.user_id is None:
|
||||
raise ValueError("user_id is required when scope_mode=single_user.")
|
||||
if self.scope_mode == "all_users" and self.user_id is not None:
|
||||
raise ValueError("user_id must be omitted when scope_mode=all_users.")
|
||||
if not self.dry_run and self.scope_mode == "all_users":
|
||||
expected = "REPAIR ALL USERS"
|
||||
provided = (self.confirmation_text or "").strip()
|
||||
if provided != expected:
|
||||
raise ValueError("confirmation_text must equal 'REPAIR ALL USERS' when applying a repair to all users.")
|
||||
return self
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksResultData(BaseModel):
|
||||
job_id: int
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationLinksEnvelope(BaseModel):
|
||||
data: AdminRepairPublicationLinksResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminNearDuplicateClusterMemberData(BaseModel):
|
||||
publication_id: int
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminNearDuplicateClusterData(BaseModel):
|
||||
cluster_key: str
|
||||
winner_publication_id: int
|
||||
member_count: int
|
||||
similarity_score: float = Field(ge=0.0, le=1.0)
|
||||
members: list[AdminNearDuplicateClusterMemberData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationNearDuplicatesRequest(BaseModel):
|
||||
dry_run: bool = True
|
||||
similarity_threshold: float = Field(default=0.78, ge=0.5, le=1.0)
|
||||
min_shared_tokens: int = Field(default=3, ge=1, le=8)
|
||||
max_year_delta: int = Field(default=1, ge=0, le=5)
|
||||
max_clusters: int = Field(default=25, ge=1, le=200)
|
||||
selected_cluster_keys: list[str] = Field(default_factory=list, max_length=200)
|
||||
requested_by: str | None = None
|
||||
confirmation_text: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_apply_mode(self) -> AdminRepairPublicationNearDuplicatesRequest:
|
||||
if self.dry_run:
|
||||
return self
|
||||
if not self.selected_cluster_keys:
|
||||
raise ValueError("selected_cluster_keys is required when dry_run=false.")
|
||||
expected = "MERGE SELECTED DUPLICATES"
|
||||
provided = (self.confirmation_text or "").strip()
|
||||
if provided != expected:
|
||||
raise ValueError(
|
||||
"confirmation_text must equal 'MERGE SELECTED DUPLICATES' when applying near-duplicate merges."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class AdminRepairPublicationNearDuplicatesResultData(BaseModel):
|
||||
job_id: int
|
||||
status: str
|
||||
scope: dict[str, Any] = Field(default_factory=dict)
|
||||
summary: dict[str, Any] = Field(default_factory=dict)
|
||||
clusters: list[AdminNearDuplicateClusterData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AdminRepairPublicationNearDuplicatesEnvelope(BaseModel):
|
||||
data: AdminRepairPublicationNearDuplicatesResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
73
app/api/schemas/auth.py
Normal file
73
app/api/schemas/auth.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from app.api.schemas.common import ApiMeta
|
||||
|
||||
|
||||
class SessionUserData(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
is_admin: bool
|
||||
is_active: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AuthMeData(BaseModel):
|
||||
authenticated: bool
|
||||
csrf_token: str
|
||||
user: SessionUserData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AuthMeEnvelope(BaseModel):
|
||||
data: AuthMeData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CsrfBootstrapData(BaseModel):
|
||||
csrf_token: str
|
||||
authenticated: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CsrfBootstrapEnvelope(BaseModel):
|
||||
data: CsrfBootstrapData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginData(BaseModel):
|
||||
authenticated: bool
|
||||
csrf_token: str
|
||||
user: SessionUserData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class LoginEnvelope(BaseModel):
|
||||
data: LoginData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
current_password: str
|
||||
new_password: str
|
||||
confirm_password: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
39
app/api/schemas/common.py
Normal file
39
app/api/schemas/common.py
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class ApiMeta(BaseModel):
|
||||
request_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ApiErrorData(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
details: Any | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ApiErrorEnvelope(BaseModel):
|
||||
error: ApiErrorData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MessageData(BaseModel):
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MessageEnvelope(BaseModel):
|
||||
data: MessageData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
151
app/api/schemas/publications.py
Normal file
151
app/api/schemas/publications.py
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.api.schemas.common import ApiMeta
|
||||
|
||||
|
||||
class DisplayIdentifierData(BaseModel):
|
||||
kind: str
|
||||
value: str
|
||||
label: str
|
||||
url: str | None
|
||||
confidence_score: float = Field(ge=0.0, le=1.0)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationItemData(BaseModel):
|
||||
publication_id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
title: str
|
||||
year: int | None
|
||||
citation_count: int
|
||||
venue_text: str | None
|
||||
pub_url: str | None
|
||||
display_identifier: DisplayIdentifierData | None = None
|
||||
pdf_url: str | None
|
||||
pdf_status: str = "untracked"
|
||||
pdf_attempt_count: int = 0
|
||||
pdf_failure_reason: str | None = None
|
||||
pdf_failure_detail: str | None = None
|
||||
is_read: bool
|
||||
is_favorite: bool = False
|
||||
first_seen_at: datetime
|
||||
is_new_in_latest_run: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationsListData(BaseModel):
|
||||
mode: str
|
||||
favorite_only: bool = False
|
||||
selected_scholar_profile_id: int | None
|
||||
unread_count: int
|
||||
favorites_count: int
|
||||
latest_count: int
|
||||
new_count: int
|
||||
total_count: int
|
||||
page: int
|
||||
page_size: int
|
||||
snapshot: str
|
||||
has_next: bool = False
|
||||
has_prev: bool = False
|
||||
publications: list[PublicationItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationsListEnvelope(BaseModel):
|
||||
data: PublicationsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkAllReadData(BaseModel):
|
||||
message: str
|
||||
updated_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkAllReadEnvelope(BaseModel):
|
||||
data: MarkAllReadData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationSelectionItem(BaseModel):
|
||||
scholar_profile_id: int
|
||||
publication_id: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadRequest(BaseModel):
|
||||
selections: list[PublicationSelectionItem] = Field(min_length=1, max_length=500)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadData(BaseModel):
|
||||
message: str
|
||||
requested_count: int
|
||||
updated_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class MarkSelectedReadEnvelope(BaseModel):
|
||||
data: MarkSelectedReadData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfRequest(BaseModel):
|
||||
scholar_profile_id: int = Field(ge=1)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfData(BaseModel):
|
||||
message: str
|
||||
queued: bool
|
||||
resolved_pdf: bool
|
||||
publication: PublicationItemData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RetryPublicationPdfEnvelope(BaseModel):
|
||||
data: RetryPublicationPdfData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteRequest(BaseModel):
|
||||
scholar_profile_id: int = Field(ge=1)
|
||||
is_favorite: bool
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteData(BaseModel):
|
||||
message: str
|
||||
publication: PublicationItemData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class TogglePublicationFavoriteEnvelope(BaseModel):
|
||||
data: TogglePublicationFavoriteData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
226
app/api/schemas/runs.py
Normal file
226
app/api/schemas/runs.py
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.api.schemas.common import ApiMeta
|
||||
|
||||
|
||||
class RunListItemData(BaseModel):
|
||||
id: int
|
||||
trigger_type: str
|
||||
status: str
|
||||
start_dt: datetime
|
||||
end_dt: datetime | None
|
||||
scholar_count: int
|
||||
new_publication_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScrapeSafetyCountersData(BaseModel):
|
||||
consecutive_blocked_runs: int = 0
|
||||
consecutive_network_runs: int = 0
|
||||
cooldown_entry_count: int = 0
|
||||
blocked_start_count: int = 0
|
||||
last_blocked_failure_count: int = 0
|
||||
last_network_failure_count: int = 0
|
||||
last_evaluated_run_id: int | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScrapeSafetyStateData(BaseModel):
|
||||
cooldown_active: bool
|
||||
cooldown_reason: str | None = None
|
||||
cooldown_reason_label: str | None = None
|
||||
cooldown_until: datetime | None = None
|
||||
cooldown_remaining_seconds: int = 0
|
||||
recommended_action: str | None = None
|
||||
counters: ScrapeSafetyCountersData = Field(default_factory=ScrapeSafetyCountersData)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunsListData(BaseModel):
|
||||
runs: list[RunListItemData]
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunsListEnvelope(BaseModel):
|
||||
data: RunsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunSummaryData(BaseModel):
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
failed_state_counts: dict[str, int] = Field(default_factory=dict)
|
||||
failed_reason_counts: dict[str, int] = Field(default_factory=dict)
|
||||
scrape_failure_counts: dict[str, int] = Field(default_factory=dict)
|
||||
retry_counts: dict[str, int] = Field(default_factory=dict)
|
||||
alert_thresholds: dict[str, int] = Field(default_factory=dict)
|
||||
alert_flags: dict[str, bool] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunAttemptLogData(BaseModel):
|
||||
attempt: int
|
||||
cstart: int
|
||||
state: str | None = None
|
||||
state_reason: str | None = None
|
||||
status_code: int | None = None
|
||||
fetch_error: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunPageLogData(BaseModel):
|
||||
page: int
|
||||
cstart: int
|
||||
state: str
|
||||
state_reason: str | None = None
|
||||
status_code: int | None = None
|
||||
publication_count: int = 0
|
||||
attempt_count: int = 0
|
||||
has_show_more_button: bool = False
|
||||
articles_range: str | None = None
|
||||
warning_codes: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDebugData(BaseModel):
|
||||
status_code: int | None = None
|
||||
final_url: str | None = None
|
||||
fetch_error: str | None = None
|
||||
body_sha256: str | None = None
|
||||
body_length: int | None = None
|
||||
has_show_more_button: bool | None = None
|
||||
articles_range: str | None = None
|
||||
state_reason: str | None = None
|
||||
warning_codes: list[str] = Field(default_factory=list)
|
||||
marker_counts_nonzero: dict[str, int] = Field(default_factory=dict)
|
||||
page_logs: list[RunPageLogData] = Field(default_factory=list)
|
||||
attempt_log: list[RunAttemptLogData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunScholarResultData(BaseModel):
|
||||
scholar_profile_id: int
|
||||
scholar_id: str
|
||||
state: str
|
||||
state_reason: str | None = None
|
||||
outcome: str
|
||||
attempt_count: int = 0
|
||||
publication_count: int = 0
|
||||
start_cstart: int = 0
|
||||
continuation_cstart: int | None = None
|
||||
continuation_enqueued: bool = False
|
||||
continuation_cleared: bool = False
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
error: str | None = None
|
||||
debug: RunDebugData | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDetailData(BaseModel):
|
||||
run: RunListItemData
|
||||
summary: RunSummaryData
|
||||
scholar_results: list[RunScholarResultData]
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RunDetailEnvelope(BaseModel):
|
||||
data: RunDetailData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ManualRunData(BaseModel):
|
||||
run_id: int
|
||||
status: str
|
||||
scholar_count: int
|
||||
succeeded_count: int
|
||||
failed_count: int
|
||||
partial_count: int
|
||||
new_publication_count: int
|
||||
reused_existing_run: bool
|
||||
idempotency_key: str | None = None
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ManualRunEnvelope(BaseModel):
|
||||
data: ManualRunData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueItemData(BaseModel):
|
||||
id: int
|
||||
scholar_profile_id: int
|
||||
scholar_label: str
|
||||
status: str
|
||||
reason: str
|
||||
dropped_reason: str | None
|
||||
attempt_count: int
|
||||
resume_cstart: int
|
||||
next_attempt_dt: datetime | None
|
||||
updated_at: datetime
|
||||
last_error: str | None
|
||||
last_run_id: int | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueListData(BaseModel):
|
||||
queue_items: list[QueueItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueListEnvelope(BaseModel):
|
||||
data: QueueListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueItemEnvelope(BaseModel):
|
||||
data: QueueItemData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueClearData(BaseModel):
|
||||
queue_item_id: int
|
||||
previous_status: str
|
||||
status: str
|
||||
message: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class QueueClearEnvelope(BaseModel):
|
||||
data: QueueClearData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
153
app/api/schemas/scholars.py
Normal file
153
app/api/schemas/scholars.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.api.schemas.common import ApiMeta
|
||||
|
||||
|
||||
class ScholarItemData(BaseModel):
|
||||
id: int
|
||||
scholar_id: str
|
||||
display_name: str | None
|
||||
profile_image_url: str | None
|
||||
profile_image_source: str
|
||||
is_enabled: bool
|
||||
baseline_completed: bool
|
||||
last_run_dt: datetime | None
|
||||
last_run_status: str | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarsListData(BaseModel):
|
||||
scholars: list[ScholarItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarsListEnvelope(BaseModel):
|
||||
data: ScholarsListData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarCreateRequest(BaseModel):
|
||||
scholar_id: str
|
||||
profile_image_url: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarEnvelope(BaseModel):
|
||||
data: ScholarItemData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchCandidateData(BaseModel):
|
||||
scholar_id: str
|
||||
display_name: str
|
||||
affiliation: str | None
|
||||
email_domain: str | None
|
||||
cited_by_count: int | None
|
||||
interests: list[str] = Field(default_factory=list)
|
||||
profile_url: str
|
||||
profile_image_url: str | None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchData(BaseModel):
|
||||
query: str
|
||||
state: str
|
||||
state_reason: str
|
||||
action_hint: str | None = None
|
||||
candidates: list[ScholarSearchCandidateData]
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarSearchEnvelope(BaseModel):
|
||||
data: ScholarSearchData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarImageUrlUpdateRequest(BaseModel):
|
||||
image_url: str
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class ScholarExportItemData(BaseModel):
|
||||
scholar_id: str
|
||||
display_name: str | None = None
|
||||
is_enabled: bool = True
|
||||
profile_image_override_url: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class PublicationExportItemData(BaseModel):
|
||||
scholar_id: str
|
||||
cluster_id: str | None = None
|
||||
fingerprint_sha256: str | None = None
|
||||
title: str
|
||||
year: int | None = None
|
||||
citation_count: int = 0
|
||||
author_text: str | None = None
|
||||
venue_text: str | None = None
|
||||
pub_url: str | None = None
|
||||
pdf_url: str | None = None
|
||||
is_read: bool = False
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataExportData(BaseModel):
|
||||
schema_version: int
|
||||
exported_at: str
|
||||
scholars: list[ScholarExportItemData]
|
||||
publications: list[PublicationExportItemData]
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataExportEnvelope(BaseModel):
|
||||
data: DataExportData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportRequest(BaseModel):
|
||||
schema_version: int | None = None
|
||||
scholars: list[ScholarExportItemData] = Field(default_factory=list)
|
||||
publications: list[PublicationExportItemData] = Field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportResultData(BaseModel):
|
||||
scholars_created: int
|
||||
scholars_updated: int
|
||||
publications_created: int
|
||||
publications_updated: int
|
||||
links_created: int
|
||||
links_updated: int
|
||||
skipped_records: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DataImportEnvelope(BaseModel):
|
||||
data: DataImportResultData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
54
app/api/schemas/settings.py
Normal file
54
app/api/schemas/settings.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from app.api.schemas.common import ApiMeta
|
||||
from app.api.schemas.runs import ScrapeSafetyStateData
|
||||
|
||||
|
||||
class SettingsPolicyData(BaseModel):
|
||||
min_run_interval_minutes: int
|
||||
min_request_delay_seconds: int
|
||||
automation_allowed: bool
|
||||
manual_run_allowed: bool
|
||||
blocked_failure_threshold: int
|
||||
network_failure_threshold: int
|
||||
cooldown_blocked_seconds: int
|
||||
cooldown_network_seconds: int
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsData(BaseModel):
|
||||
auto_run_enabled: bool
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
nav_visible_pages: list[str]
|
||||
policy: SettingsPolicyData
|
||||
safety_state: ScrapeSafetyStateData
|
||||
|
||||
openalex_api_key: str | None = None
|
||||
crossref_api_token: str | None = None
|
||||
crossref_api_mailto: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsEnvelope(BaseModel):
|
||||
data: SettingsData
|
||||
meta: ApiMeta
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class SettingsUpdateRequest(BaseModel):
|
||||
auto_run_enabled: bool
|
||||
run_interval_minutes: int
|
||||
request_delay_seconds: int
|
||||
nav_visible_pages: list[str] | None = None
|
||||
|
||||
openalex_api_key: str | None = None
|
||||
crossref_api_token: str | None = None
|
||||
crossref_api_mailto: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
File diff suppressed because it is too large
Load diff
323
app/services/ingestion/enrichment.py
Normal file
323
app/services/ingestion/enrichment.py
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
Publication,
|
||||
RunStatus,
|
||||
ScholarProfile,
|
||||
ScholarPublication,
|
||||
)
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
from app.services.runs.events import run_events
|
||||
from app.services.scholar.parser import PublicationCandidate
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EnrichmentRunner:
|
||||
"""Post-run OpenAlex enrichment logic.
|
||||
|
||||
Receives service dependencies at construction so it can be tested
|
||||
independently of ``ScholarIngestionService``.
|
||||
"""
|
||||
|
||||
async def run_is_canceled(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run_id: int,
|
||||
) -> bool:
|
||||
check_result = await db_session.execute(select(CrawlRun.status).where(CrawlRun.id == run_id))
|
||||
status = check_result.scalar_one_or_none()
|
||||
if status is None:
|
||||
raise RuntimeError(f"Missing crawl_run for run_id={run_id}.")
|
||||
return status == RunStatus.CANCELED
|
||||
|
||||
async def enrich_pending_publications(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run_id: int,
|
||||
openalex_api_key: str | None = None,
|
||||
) -> None:
|
||||
"""Enrich unenriched publications with OpenAlex data.
|
||||
|
||||
Stops immediately on budget exhaustion (429 with $0 remaining).
|
||||
Sleeps 60s and continues on transient rate limits.
|
||||
"""
|
||||
from app.services.openalex.client import (
|
||||
OpenAlexBudgetExhaustedError,
|
||||
OpenAlexClient,
|
||||
OpenAlexRateLimitError,
|
||||
)
|
||||
from app.services.openalex.matching import find_best_match
|
||||
|
||||
run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
|
||||
user_id = run_result.scalar_one()
|
||||
|
||||
now = datetime.now(UTC)
|
||||
cooldown_threshold = now - timedelta(days=7)
|
||||
|
||||
stmt = (
|
||||
select(Publication)
|
||||
.join(ScholarPublication)
|
||||
.join(ScholarProfile, ScholarPublication.scholar_profile_id == ScholarProfile.id)
|
||||
.where(
|
||||
ScholarProfile.user_id == user_id,
|
||||
Publication.openalex_enriched.is_(False),
|
||||
or_(
|
||||
Publication.openalex_last_attempt_at.is_(None),
|
||||
Publication.openalex_last_attempt_at < cooldown_threshold,
|
||||
),
|
||||
)
|
||||
.distinct()
|
||||
)
|
||||
result = await db_session.execute(stmt)
|
||||
publications = list(result.scalars().all())
|
||||
|
||||
if not publications:
|
||||
return
|
||||
|
||||
resolved_key = openalex_api_key or settings.openalex_api_key
|
||||
client = OpenAlexClient(api_key=resolved_key, mailto=settings.crossref_api_mailto)
|
||||
batch_size = 25
|
||||
arxiv_lookup_allowed = True
|
||||
|
||||
for i in range(0, len(publications), batch_size):
|
||||
if await self.run_is_canceled(db_session, run_id=run_id):
|
||||
logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id})
|
||||
return
|
||||
batch = publications[i : i + batch_size]
|
||||
titles = [
|
||||
" ".join(re.sub(r"[^\w\s]", " ", p.title_raw).split())
|
||||
for p in batch
|
||||
if p.title_raw and p.title_raw.strip()
|
||||
]
|
||||
|
||||
if not titles:
|
||||
continue
|
||||
|
||||
try:
|
||||
openalex_works = await client.get_works_by_filter(
|
||||
{"title.search": "|".join(titles)}, limit=batch_size * 3
|
||||
)
|
||||
except OpenAlexBudgetExhaustedError:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.openalex_budget_exhausted",
|
||||
run_id=run_id,
|
||||
)
|
||||
break
|
||||
except OpenAlexRateLimitError:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.openalex_rate_limited",
|
||||
run_id=run_id,
|
||||
)
|
||||
await asyncio.sleep(60)
|
||||
continue
|
||||
except Exception as e:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.openalex_enrichment_failed",
|
||||
error=str(e),
|
||||
run_id=run_id,
|
||||
)
|
||||
continue
|
||||
|
||||
for p in batch:
|
||||
if await self.run_is_canceled(db_session, run_id=run_id):
|
||||
logger.info("ingestion.enrichment_aborted", extra={"run_id": run_id})
|
||||
return
|
||||
|
||||
p.openalex_last_attempt_at = now
|
||||
arxiv_lookup_allowed = await self._discover_identifiers_for_enrichment(
|
||||
db_session,
|
||||
publication=p,
|
||||
run_id=run_id,
|
||||
allow_arxiv_lookup=arxiv_lookup_allowed,
|
||||
)
|
||||
|
||||
match = find_best_match(
|
||||
target_title=p.title_raw,
|
||||
target_year=p.year,
|
||||
target_authors=p.author_text or "",
|
||||
candidates=openalex_works,
|
||||
)
|
||||
if match:
|
||||
p.year = match.publication_year or p.year
|
||||
p.citation_count = match.cited_by_count or p.citation_count
|
||||
p.pdf_url = match.oa_url or p.pdf_url
|
||||
p.openalex_enriched = True
|
||||
|
||||
await db_session.flush()
|
||||
|
||||
from app.services.publications.dedup import sweep_identifier_duplicates
|
||||
|
||||
merge_count = await sweep_identifier_duplicates(db_session)
|
||||
if merge_count:
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"ingestion.identifier_dedup_sweep",
|
||||
merged_count=merge_count,
|
||||
run_id=run_id,
|
||||
)
|
||||
|
||||
async def _discover_identifiers_for_enrichment(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
publication: Publication,
|
||||
run_id: int,
|
||||
allow_arxiv_lookup: bool,
|
||||
) -> bool:
|
||||
if not allow_arxiv_lookup:
|
||||
await identifier_service.sync_identifiers_for_publication_fields(
|
||||
db_session,
|
||||
publication=publication,
|
||||
)
|
||||
await self._publish_identifier_update_event(
|
||||
db_session,
|
||||
run_id=run_id,
|
||||
publication_id=int(publication.id),
|
||||
)
|
||||
return False
|
||||
try:
|
||||
await identifier_service.discover_and_sync_identifiers_for_publication(
|
||||
db_session,
|
||||
publication=publication,
|
||||
scholar_label=publication.author_text or "",
|
||||
)
|
||||
await self._publish_identifier_update_event(
|
||||
db_session,
|
||||
run_id=run_id,
|
||||
publication_id=int(publication.id),
|
||||
)
|
||||
return True
|
||||
except ArxivRateLimitError:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.arxiv_rate_limited",
|
||||
run_id=run_id,
|
||||
publication_id=int(publication.id),
|
||||
detail="arXiv temporarily disabled for remaining enrichment pass",
|
||||
)
|
||||
await identifier_service.sync_identifiers_for_publication_fields(
|
||||
db_session,
|
||||
publication=publication,
|
||||
)
|
||||
await self._publish_identifier_update_event(
|
||||
db_session,
|
||||
run_id=run_id,
|
||||
publication_id=int(publication.id),
|
||||
)
|
||||
return False
|
||||
|
||||
async def _publish_identifier_update_event(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run_id: int,
|
||||
publication_id: int,
|
||||
) -> None:
|
||||
display = await identifier_service.display_identifier_for_publication_id(
|
||||
db_session,
|
||||
publication_id=publication_id,
|
||||
)
|
||||
if display is None:
|
||||
return
|
||||
await run_events.publish(
|
||||
run_id=run_id,
|
||||
event_type="identifier_updated",
|
||||
data={
|
||||
"publication_id": int(publication_id),
|
||||
"display_identifier": {
|
||||
"kind": display.kind,
|
||||
"value": display.value,
|
||||
"label": display.label,
|
||||
"url": display.url,
|
||||
"confidence_score": float(display.confidence_score),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
async def enrich_publications_with_openalex(
|
||||
self,
|
||||
scholar: ScholarProfile,
|
||||
publications: list[PublicationCandidate],
|
||||
) -> list[PublicationCandidate]:
|
||||
if not publications:
|
||||
return publications
|
||||
|
||||
from app.services.openalex.client import OpenAlexClient
|
||||
from app.services.openalex.matching import find_best_match
|
||||
|
||||
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
|
||||
|
||||
batch_size = 25
|
||||
enriched: list[PublicationCandidate] = []
|
||||
|
||||
for i in range(0, len(publications), batch_size):
|
||||
batch = publications[i : i + batch_size]
|
||||
|
||||
titles = []
|
||||
for p in batch:
|
||||
if not p.title:
|
||||
continue
|
||||
safe_title = re.sub(r"[^\w\s]", " ", p.title)
|
||||
safe_title = " ".join(safe_title.split())
|
||||
if safe_title:
|
||||
titles.append(safe_title)
|
||||
|
||||
if not titles:
|
||||
enriched.extend(batch)
|
||||
continue
|
||||
|
||||
query = "|".join(t for t in titles)
|
||||
try:
|
||||
openalex_works = await client.get_works_by_filter({"title.search": query}, limit=batch_size * 3)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"ingestion.openalex_enrichment_failed",
|
||||
extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id},
|
||||
)
|
||||
openalex_works = []
|
||||
|
||||
for p in batch:
|
||||
match = find_best_match(
|
||||
target_title=p.title,
|
||||
target_year=p.year,
|
||||
target_authors=p.authors_text or (scholar.display_name or scholar.scholar_id),
|
||||
candidates=openalex_works,
|
||||
)
|
||||
if match:
|
||||
new_p = PublicationCandidate(
|
||||
title=p.title,
|
||||
title_url=p.title_url,
|
||||
cluster_id=p.cluster_id,
|
||||
year=match.publication_year or p.year,
|
||||
citation_count=match.cited_by_count,
|
||||
authors_text=p.authors_text,
|
||||
venue_text=p.venue_text,
|
||||
pdf_url=match.oa_url or p.pdf_url,
|
||||
)
|
||||
enriched.append(new_p)
|
||||
else:
|
||||
enriched.append(p)
|
||||
return enriched
|
||||
417
app/services/ingestion/run_completion.py
Normal file
417
app/services/ingestion/run_completion.py
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
RunStatus,
|
||||
ScholarProfile,
|
||||
)
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.ingestion import safety as run_safety_service
|
||||
from app.services.ingestion.constants import (
|
||||
FAILED_STATES,
|
||||
FAILURE_BUCKET_BLOCKED,
|
||||
FAILURE_BUCKET_INGESTION,
|
||||
FAILURE_BUCKET_LAYOUT,
|
||||
FAILURE_BUCKET_NETWORK,
|
||||
FAILURE_BUCKET_OTHER,
|
||||
)
|
||||
from app.services.ingestion.fingerprints import _build_body_excerpt
|
||||
from app.services.ingestion.types import (
|
||||
RunAlertSummary,
|
||||
RunExecutionSummary,
|
||||
RunFailureSummary,
|
||||
RunProgress,
|
||||
ScholarProcessingOutcome,
|
||||
)
|
||||
from app.services.scholar.parser import ParsedProfilePage, ParseState
|
||||
from app.services.scholar.source import FetchResult
|
||||
from app.settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _int_or_default(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def classify_failure_bucket(*, state: str, state_reason: str) -> str:
|
||||
reason = state_reason.strip().lower()
|
||||
normalized_state = state.strip().lower()
|
||||
|
||||
if normalized_state == ParseState.BLOCKED_OR_CAPTCHA.value or reason.startswith("blocked_"):
|
||||
return FAILURE_BUCKET_BLOCKED
|
||||
if normalized_state == ParseState.NETWORK_ERROR.value or reason.startswith("network_"):
|
||||
return FAILURE_BUCKET_NETWORK
|
||||
if normalized_state == ParseState.LAYOUT_CHANGED.value:
|
||||
return FAILURE_BUCKET_LAYOUT
|
||||
if normalized_state == "ingestion_error":
|
||||
return FAILURE_BUCKET_INGESTION
|
||||
return FAILURE_BUCKET_OTHER
|
||||
|
||||
|
||||
def summarize_failures(
|
||||
*,
|
||||
scholar_results: list[dict[str, Any]],
|
||||
) -> RunFailureSummary:
|
||||
failed_state_counts: dict[str, int] = {}
|
||||
failed_reason_counts: dict[str, int] = {}
|
||||
scrape_failure_counts: dict[str, int] = {}
|
||||
retries_scheduled_count = 0
|
||||
scholars_with_retries_count = 0
|
||||
retry_exhausted_count = 0
|
||||
for entry in scholar_results:
|
||||
retries_for_entry = max(0, _int_or_default(entry.get("attempt_count"), 0) - 1)
|
||||
if retries_for_entry > 0:
|
||||
retries_scheduled_count += retries_for_entry
|
||||
scholars_with_retries_count += 1
|
||||
if str(entry.get("outcome", "")) != "failed":
|
||||
continue
|
||||
state = str(entry.get("state", "")).strip()
|
||||
if state not in FAILED_STATES:
|
||||
continue
|
||||
failed_state_counts[state] = failed_state_counts.get(state, 0) + 1
|
||||
reason = str(entry.get("state_reason", "")).strip()
|
||||
if reason:
|
||||
failed_reason_counts[reason] = failed_reason_counts.get(reason, 0) + 1
|
||||
bucket = classify_failure_bucket(state=state, state_reason=reason)
|
||||
scrape_failure_counts[bucket] = scrape_failure_counts.get(bucket, 0) + 1
|
||||
if state == ParseState.NETWORK_ERROR.value and retries_for_entry > 0:
|
||||
retry_exhausted_count += 1
|
||||
return RunFailureSummary(
|
||||
failed_state_counts=failed_state_counts,
|
||||
failed_reason_counts=failed_reason_counts,
|
||||
scrape_failure_counts=scrape_failure_counts,
|
||||
retries_scheduled_count=retries_scheduled_count,
|
||||
scholars_with_retries_count=scholars_with_retries_count,
|
||||
retry_exhausted_count=retry_exhausted_count,
|
||||
)
|
||||
|
||||
|
||||
def build_alert_summary(
|
||||
*,
|
||||
failure_summary: RunFailureSummary,
|
||||
alert_blocked_failure_threshold: int,
|
||||
alert_network_failure_threshold: int,
|
||||
alert_retry_scheduled_threshold: int,
|
||||
) -> RunAlertSummary:
|
||||
blocked_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_BLOCKED, 0))
|
||||
network_failure_count = int(failure_summary.scrape_failure_counts.get(FAILURE_BUCKET_NETWORK, 0))
|
||||
blocked_threshold = max(1, int(alert_blocked_failure_threshold))
|
||||
network_threshold = max(1, int(alert_network_failure_threshold))
|
||||
retry_threshold = max(1, int(alert_retry_scheduled_threshold))
|
||||
alert_flags = {
|
||||
"blocked_failure_threshold_exceeded": blocked_failure_count >= blocked_threshold,
|
||||
"network_failure_threshold_exceeded": network_failure_count >= network_threshold,
|
||||
"retry_scheduled_threshold_exceeded": failure_summary.retries_scheduled_count >= retry_threshold,
|
||||
}
|
||||
return RunAlertSummary(
|
||||
blocked_failure_count=blocked_failure_count,
|
||||
network_failure_count=network_failure_count,
|
||||
blocked_failure_threshold=blocked_threshold,
|
||||
network_failure_threshold=network_threshold,
|
||||
retry_scheduled_threshold=retry_threshold,
|
||||
alert_flags=alert_flags,
|
||||
)
|
||||
|
||||
|
||||
def apply_safety_outcome(
|
||||
*,
|
||||
user_settings: Any,
|
||||
run: CrawlRun,
|
||||
user_id: int,
|
||||
alert_summary: RunAlertSummary,
|
||||
) -> None:
|
||||
pre_apply_state = run_safety_service.get_safety_event_context(
|
||||
user_settings,
|
||||
now_utc=datetime.now(UTC),
|
||||
)
|
||||
safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome(
|
||||
user_settings,
|
||||
run_id=int(run.id),
|
||||
blocked_failure_count=alert_summary.blocked_failure_count,
|
||||
network_failure_count=alert_summary.network_failure_count,
|
||||
blocked_failure_threshold=alert_summary.blocked_failure_threshold,
|
||||
network_failure_threshold=alert_summary.network_failure_threshold,
|
||||
blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
|
||||
network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
|
||||
now_utc=datetime.now(UTC),
|
||||
)
|
||||
if cooldown_trigger_reason is not None:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.safety_cooldown_entered",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
reason=cooldown_trigger_reason,
|
||||
blocked_failure_count=alert_summary.blocked_failure_count,
|
||||
network_failure_count=alert_summary.network_failure_count,
|
||||
blocked_failure_threshold=alert_summary.blocked_failure_threshold,
|
||||
network_failure_threshold=alert_summary.network_failure_threshold,
|
||||
cooldown_until=safety_state.get("cooldown_until"),
|
||||
cooldown_remaining_seconds=safety_state.get("cooldown_remaining_seconds"),
|
||||
safety_counters=safety_state.get("counters", {}),
|
||||
)
|
||||
elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"):
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"ingestion.cooldown_cleared",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
reason=pre_apply_state.get("cooldown_reason"),
|
||||
cooldown_until=pre_apply_state.get("cooldown_until"),
|
||||
)
|
||||
|
||||
|
||||
def finalize_run_record(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholars: list[ScholarProfile],
|
||||
progress: RunProgress,
|
||||
failure_summary: RunFailureSummary,
|
||||
alert_summary: RunAlertSummary,
|
||||
idempotency_key: str | None,
|
||||
run_status: RunStatus,
|
||||
) -> None:
|
||||
run.end_dt = datetime.now(UTC)
|
||||
if run.status != RunStatus.CANCELED:
|
||||
run.status = run_status
|
||||
run.error_log = {
|
||||
"scholar_results": progress.scholar_results,
|
||||
"summary": {
|
||||
"succeeded_count": progress.succeeded_count,
|
||||
"failed_count": progress.failed_count,
|
||||
"partial_count": progress.partial_count,
|
||||
"failed_state_counts": failure_summary.failed_state_counts,
|
||||
"failed_reason_counts": failure_summary.failed_reason_counts,
|
||||
"scrape_failure_counts": failure_summary.scrape_failure_counts,
|
||||
"retry_counts": {
|
||||
"retries_scheduled_count": failure_summary.retries_scheduled_count,
|
||||
"scholars_with_retries_count": failure_summary.scholars_with_retries_count,
|
||||
"retry_exhausted_count": failure_summary.retry_exhausted_count,
|
||||
},
|
||||
"alert_thresholds": {
|
||||
"blocked_failure_threshold": alert_summary.blocked_failure_threshold,
|
||||
"network_failure_threshold": alert_summary.network_failure_threshold,
|
||||
"retry_scheduled_threshold": alert_summary.retry_scheduled_threshold,
|
||||
},
|
||||
"alert_flags": alert_summary.alert_flags,
|
||||
},
|
||||
"meta": {"idempotency_key": idempotency_key} if idempotency_key else {},
|
||||
}
|
||||
|
||||
|
||||
def resolve_run_status(
|
||||
*,
|
||||
scholar_count: int,
|
||||
succeeded_count: int,
|
||||
failed_count: int,
|
||||
partial_count: int,
|
||||
) -> RunStatus:
|
||||
if scholar_count == 0:
|
||||
return RunStatus.SUCCESS
|
||||
if failed_count == scholar_count:
|
||||
return RunStatus.FAILED
|
||||
if failed_count > 0 or partial_count > 0:
|
||||
return RunStatus.PARTIAL_FAILURE
|
||||
if succeeded_count > 0:
|
||||
return RunStatus.SUCCESS
|
||||
return RunStatus.FAILED
|
||||
|
||||
|
||||
def build_failure_debug_context(
|
||||
*,
|
||||
fetch_result: FetchResult,
|
||||
parsed_page: ParsedProfilePage,
|
||||
attempt_log: list[dict[str, Any]],
|
||||
page_logs: list[dict[str, Any]] | None = None,
|
||||
exception: Exception | None = None,
|
||||
) -> dict[str, Any]:
|
||||
context: dict[str, Any] = {
|
||||
"requested_url": fetch_result.requested_url,
|
||||
"final_url": fetch_result.final_url,
|
||||
"status_code": fetch_result.status_code,
|
||||
"fetch_error": fetch_result.error,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"profile_name": parsed_page.profile_name,
|
||||
"profile_image_url": parsed_page.profile_image_url,
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"has_operation_error_banner": parsed_page.has_operation_error_banner,
|
||||
"warning_codes": parsed_page.warnings,
|
||||
"marker_counts_nonzero": {key: value for key, value in parsed_page.marker_counts.items() if value > 0},
|
||||
"body_length": len(fetch_result.body),
|
||||
"body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() if fetch_result.body else None,
|
||||
"body_excerpt": _build_body_excerpt(fetch_result.body),
|
||||
"attempt_log": attempt_log,
|
||||
}
|
||||
if page_logs:
|
||||
context["page_logs"] = page_logs
|
||||
if exception is not None:
|
||||
context["exception_type"] = type(exception).__name__
|
||||
context["exception_message"] = str(exception)
|
||||
return context
|
||||
|
||||
|
||||
def complete_run_for_user(
|
||||
*,
|
||||
user_settings: Any,
|
||||
run: CrawlRun,
|
||||
scholars: list[ScholarProfile],
|
||||
user_id: int,
|
||||
progress: RunProgress,
|
||||
idempotency_key: str | None,
|
||||
alert_blocked_failure_threshold: int,
|
||||
alert_network_failure_threshold: int,
|
||||
alert_retry_scheduled_threshold: int,
|
||||
) -> tuple[RunFailureSummary, RunAlertSummary]:
|
||||
failure_summary = summarize_failures(scholar_results=progress.scholar_results)
|
||||
alert_summary = build_alert_summary(
|
||||
failure_summary=failure_summary,
|
||||
alert_blocked_failure_threshold=alert_blocked_failure_threshold,
|
||||
alert_network_failure_threshold=alert_network_failure_threshold,
|
||||
alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
|
||||
)
|
||||
if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.alert_blocked_failure_threshold_exceeded",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
blocked_failure_count=alert_summary.blocked_failure_count,
|
||||
threshold=alert_summary.blocked_failure_threshold,
|
||||
)
|
||||
if alert_summary.alert_flags["network_failure_threshold_exceeded"]:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.alert_network_failure_threshold_exceeded",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
network_failure_count=alert_summary.network_failure_count,
|
||||
threshold=alert_summary.network_failure_threshold,
|
||||
)
|
||||
if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]:
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.alert_retry_scheduled_threshold_exceeded",
|
||||
user_id=user_id,
|
||||
crawl_run_id=int(run.id),
|
||||
retries_scheduled_count=failure_summary.retries_scheduled_count,
|
||||
threshold=alert_summary.retry_scheduled_threshold,
|
||||
)
|
||||
apply_safety_outcome(user_settings=user_settings, run=run, user_id=user_id, alert_summary=alert_summary)
|
||||
run_status = resolve_run_status(
|
||||
scholar_count=len(scholars),
|
||||
succeeded_count=progress.succeeded_count,
|
||||
failed_count=progress.failed_count,
|
||||
partial_count=progress.partial_count,
|
||||
)
|
||||
finalize_run_record(
|
||||
run=run,
|
||||
scholars=scholars,
|
||||
progress=progress,
|
||||
failure_summary=failure_summary,
|
||||
alert_summary=alert_summary,
|
||||
idempotency_key=idempotency_key,
|
||||
run_status=run_status,
|
||||
)
|
||||
return failure_summary, alert_summary
|
||||
|
||||
|
||||
def result_counters(result_entry: dict[str, Any]) -> tuple[int, int, int]:
|
||||
outcome = str(result_entry.get("outcome", "")).strip().lower()
|
||||
if outcome == "success":
|
||||
return 1, 0, 0
|
||||
if outcome == "partial":
|
||||
return 1, 0, 1
|
||||
if outcome == "failed":
|
||||
return 0, 1, 0
|
||||
raise RuntimeError(f"Unexpected scholar outcome label: {outcome!r}")
|
||||
|
||||
|
||||
def find_scholar_result_index(
|
||||
*,
|
||||
scholar_results: list[dict[str, Any]],
|
||||
scholar_profile_id: int,
|
||||
) -> int | None:
|
||||
for index, result_entry in enumerate(scholar_results):
|
||||
current_scholar_id = _int_or_default(result_entry.get("scholar_profile_id"), 0)
|
||||
if current_scholar_id == scholar_profile_id:
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def adjust_progress_counts(
|
||||
*,
|
||||
progress: RunProgress,
|
||||
succeeded_delta: int,
|
||||
failed_delta: int,
|
||||
partial_delta: int,
|
||||
) -> None:
|
||||
progress.succeeded_count += succeeded_delta
|
||||
progress.failed_count += failed_delta
|
||||
progress.partial_count += partial_delta
|
||||
if progress.succeeded_count < 0 or progress.failed_count < 0 or progress.partial_count < 0:
|
||||
raise RuntimeError("RunProgress counters entered invalid negative state.")
|
||||
|
||||
|
||||
def apply_outcome_to_progress(
|
||||
*,
|
||||
progress: RunProgress,
|
||||
outcome: ScholarProcessingOutcome,
|
||||
) -> None:
|
||||
scholar_profile_id = _int_or_default(outcome.result_entry.get("scholar_profile_id"), 0)
|
||||
if scholar_profile_id <= 0:
|
||||
raise RuntimeError("Scholar outcome missing valid scholar_profile_id.")
|
||||
prior_index = find_scholar_result_index(
|
||||
scholar_results=progress.scholar_results,
|
||||
scholar_profile_id=scholar_profile_id,
|
||||
)
|
||||
next_succeeded, next_failed, next_partial = result_counters(outcome.result_entry)
|
||||
if prior_index is None:
|
||||
progress.scholar_results.append(outcome.result_entry)
|
||||
adjust_progress_counts(
|
||||
progress=progress,
|
||||
succeeded_delta=next_succeeded,
|
||||
failed_delta=next_failed,
|
||||
partial_delta=next_partial,
|
||||
)
|
||||
return
|
||||
previous_entry = progress.scholar_results[prior_index]
|
||||
prev_succeeded, prev_failed, prev_partial = result_counters(previous_entry)
|
||||
progress.scholar_results[prior_index] = outcome.result_entry
|
||||
adjust_progress_counts(
|
||||
progress=progress,
|
||||
succeeded_delta=next_succeeded - prev_succeeded,
|
||||
failed_delta=next_failed - prev_failed,
|
||||
partial_delta=next_partial - prev_partial,
|
||||
)
|
||||
|
||||
|
||||
def run_execution_summary(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholars: list[ScholarProfile],
|
||||
progress: RunProgress,
|
||||
) -> RunExecutionSummary:
|
||||
return RunExecutionSummary(
|
||||
crawl_run_id=run.id,
|
||||
status=run.status,
|
||||
scholar_count=len(scholars),
|
||||
succeeded_count=progress.succeeded_count,
|
||||
failed_count=progress.failed_count,
|
||||
partial_count=progress.partial_count,
|
||||
new_publication_count=run.new_pub_count,
|
||||
)
|
||||
614
app/services/ingestion/scholar_processing.py
Normal file
614
app/services/ingestion/scholar_processing.py
Normal file
|
|
@ -0,0 +1,614 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import (
|
||||
CrawlRun,
|
||||
RunStatus,
|
||||
ScholarProfile,
|
||||
)
|
||||
from app.logging_utils import structured_log
|
||||
from app.services.ingestion import queue as queue_service
|
||||
from app.services.ingestion.constants import (
|
||||
RESUMABLE_PARTIAL_REASON_PREFIXES,
|
||||
RESUMABLE_PARTIAL_REASONS,
|
||||
)
|
||||
from app.services.ingestion.pagination import PaginationEngine
|
||||
from app.services.ingestion.publication_upsert import upsert_profile_publications
|
||||
from app.services.ingestion.run_completion import apply_outcome_to_progress, build_failure_debug_context
|
||||
from app.services.ingestion.types import (
|
||||
PagedParseResult,
|
||||
RunProgress,
|
||||
ScholarProcessingOutcome,
|
||||
)
|
||||
from app.services.scholar.parser import ParseState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def assert_valid_paged_parse_result(
|
||||
*,
|
||||
scholar_id: str,
|
||||
paged_parse_result: PagedParseResult,
|
||||
) -> None:
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
if parsed_page.state in {ParseState.OK, ParseState.NO_RESULTS} and any(
|
||||
code.startswith("layout_") for code in parsed_page.warnings
|
||||
):
|
||||
raise RuntimeError(f"Layout warning marked as terminal for scholar_id={scholar_id}.")
|
||||
for publication in paged_parse_result.publications:
|
||||
if not publication.title.strip():
|
||||
raise RuntimeError(f"Malformed publication title for scholar_id={scholar_id}.")
|
||||
if publication.citation_count is not None and int(publication.citation_count) < 0:
|
||||
raise RuntimeError(f"Negative citation count for scholar_id={scholar_id}.")
|
||||
|
||||
|
||||
def apply_first_page_profile_metadata(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
paged_parse_result: PagedParseResult,
|
||||
run_dt: datetime,
|
||||
) -> None:
|
||||
first_page = paged_parse_result.first_page_parsed_page
|
||||
if first_page.profile_name and not (scholar.display_name or "").strip():
|
||||
scholar.display_name = first_page.profile_name
|
||||
if first_page.profile_image_url:
|
||||
scholar.profile_image_url = first_page.profile_image_url
|
||||
scholar.last_initial_page_checked_at = run_dt
|
||||
|
||||
|
||||
def build_result_entry(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
start_cstart: int,
|
||||
paged_parse_result: PagedParseResult,
|
||||
) -> dict[str, Any]:
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
return {
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
"state": parsed_page.state.value,
|
||||
"state_reason": parsed_page.state_reason,
|
||||
"outcome": "failed",
|
||||
"attempt_count": len(paged_parse_result.attempt_log),
|
||||
"publication_count": len(paged_parse_result.publications),
|
||||
"start_cstart": start_cstart,
|
||||
"articles_range": parsed_page.articles_range,
|
||||
"warnings": parsed_page.warnings,
|
||||
"has_show_more_button": parsed_page.has_show_more_button,
|
||||
"pages_fetched": paged_parse_result.pages_fetched,
|
||||
"pages_attempted": paged_parse_result.pages_attempted,
|
||||
"has_more_remaining": paged_parse_result.has_more_remaining,
|
||||
"pagination_truncated_reason": paged_parse_result.pagination_truncated_reason,
|
||||
"continuation_cstart": paged_parse_result.continuation_cstart,
|
||||
"skipped_no_change": paged_parse_result.skipped_no_change,
|
||||
"initial_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
|
||||
}
|
||||
|
||||
|
||||
def skipped_no_change_outcome(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
) -> ScholarProcessingOutcome:
|
||||
first_page = paged_parse_result.first_page_parsed_page
|
||||
scholar.last_run_status = RunStatus.SUCCESS
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["state"] = first_page.state.value
|
||||
result_entry["state_reason"] = "no_change_initial_page_signature"
|
||||
result_entry["outcome"] = "success"
|
||||
result_entry["publication_count"] = 0
|
||||
result_entry["warnings"] = first_page.warnings
|
||||
result_entry["debug"] = {
|
||||
"state_reason": "no_change_initial_page_signature",
|
||||
"first_page_fingerprint_sha256": paged_parse_result.first_page_fingerprint_sha256,
|
||||
"attempt_log": paged_parse_result.attempt_log,
|
||||
"page_logs": paged_parse_result.page_logs,
|
||||
}
|
||||
return ScholarProcessingOutcome(
|
||||
result_entry=result_entry,
|
||||
succeeded_count_delta=1,
|
||||
failed_count_delta=0,
|
||||
partial_count_delta=0,
|
||||
discovered_publication_count=0,
|
||||
)
|
||||
|
||||
|
||||
async def upsert_publications_outcome(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
) -> ScholarProcessingOutcome:
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
publications = paged_parse_result.publications
|
||||
had_page_failure = parsed_page.state not in {ParseState.OK, ParseState.NO_RESULTS}
|
||||
has_partial_set = len(publications) > 0 and had_page_failure
|
||||
if (not had_page_failure) or has_partial_set:
|
||||
return await _upsert_success_or_exception(
|
||||
db_session,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
has_partial_publication_set=has_partial_set,
|
||||
)
|
||||
return _parse_failure_outcome(
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
)
|
||||
|
||||
|
||||
async def _upsert_success_or_exception(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
has_partial_publication_set: bool,
|
||||
) -> ScholarProcessingOutcome:
|
||||
try:
|
||||
return _upsert_success(
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
has_partial_publication_set=has_partial_publication_set,
|
||||
)
|
||||
except Exception as exc:
|
||||
return _upsert_exception_outcome(
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
exc=exc,
|
||||
)
|
||||
|
||||
|
||||
def _upsert_success(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
has_partial_publication_set: bool,
|
||||
) -> ScholarProcessingOutcome:
|
||||
discovered_count = paged_parse_result.discovered_publication_count
|
||||
is_partial = (
|
||||
paged_parse_result.has_more_remaining
|
||||
or paged_parse_result.pagination_truncated_reason is not None
|
||||
or has_partial_publication_set
|
||||
)
|
||||
scholar.last_run_status = RunStatus.PARTIAL_FAILURE if is_partial else RunStatus.SUCCESS
|
||||
scholar.last_run_dt = run_dt
|
||||
if not is_partial and paged_parse_result.first_page_fingerprint_sha256:
|
||||
scholar.last_initial_page_fingerprint_sha256 = paged_parse_result.first_page_fingerprint_sha256
|
||||
result_entry["outcome"] = "partial" if is_partial else "success"
|
||||
if is_partial:
|
||||
result_entry["debug"] = build_failure_debug_context(
|
||||
fetch_result=paged_parse_result.fetch_result,
|
||||
parsed_page=paged_parse_result.parsed_page,
|
||||
attempt_log=paged_parse_result.attempt_log,
|
||||
page_logs=paged_parse_result.page_logs,
|
||||
)
|
||||
return ScholarProcessingOutcome(
|
||||
result_entry=result_entry,
|
||||
succeeded_count_delta=1,
|
||||
failed_count_delta=0,
|
||||
partial_count_delta=1 if is_partial else 0,
|
||||
discovered_publication_count=discovered_count,
|
||||
)
|
||||
|
||||
|
||||
def _upsert_exception_outcome(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
exc: Exception,
|
||||
) -> ScholarProcessingOutcome:
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["state"] = "ingestion_error"
|
||||
result_entry["state_reason"] = "publication_upsert_exception"
|
||||
result_entry["outcome"] = "failed"
|
||||
result_entry["error"] = str(exc)
|
||||
result_entry["debug"] = build_failure_debug_context(
|
||||
fetch_result=paged_parse_result.fetch_result,
|
||||
parsed_page=paged_parse_result.parsed_page,
|
||||
attempt_log=paged_parse_result.attempt_log,
|
||||
page_logs=paged_parse_result.page_logs,
|
||||
exception=exc,
|
||||
)
|
||||
logger.exception(
|
||||
"ingestion.scholar_failed",
|
||||
extra={
|
||||
"crawl_run_id": run.id,
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
},
|
||||
)
|
||||
return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
|
||||
|
||||
|
||||
def _parse_failure_outcome(
|
||||
*,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
) -> ScholarProcessingOutcome:
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = run_dt
|
||||
result_entry["debug"] = build_failure_debug_context(
|
||||
fetch_result=paged_parse_result.fetch_result,
|
||||
parsed_page=paged_parse_result.parsed_page,
|
||||
attempt_log=paged_parse_result.attempt_log,
|
||||
page_logs=paged_parse_result.page_logs,
|
||||
)
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"ingestion.scholar_parse_failed",
|
||||
scholar_profile_id=scholar.id,
|
||||
scholar_id=scholar.scholar_id,
|
||||
state=paged_parse_result.parsed_page.state.value,
|
||||
state_reason=paged_parse_result.parsed_page.state_reason,
|
||||
status_code=paged_parse_result.fetch_result.status_code,
|
||||
)
|
||||
return ScholarProcessingOutcome(result_entry, 0, 1, 0, 0)
|
||||
|
||||
|
||||
async def sync_continuation_queue(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
user_id: int,
|
||||
scholar: ScholarProfile,
|
||||
run: CrawlRun,
|
||||
start_cstart: int,
|
||||
result_entry: dict[str, Any],
|
||||
paged_parse_result: PagedParseResult,
|
||||
auto_queue_continuations: bool,
|
||||
queue_delay_seconds: int,
|
||||
) -> None:
|
||||
queue_reason, queue_cstart = resolve_continuation_queue_target(
|
||||
outcome=str(result_entry.get("outcome", "")),
|
||||
state=str(result_entry.get("state", "")),
|
||||
pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
|
||||
continuation_cstart=paged_parse_result.continuation_cstart,
|
||||
fallback_cstart=start_cstart,
|
||||
)
|
||||
if auto_queue_continuations and queue_reason is not None:
|
||||
await queue_service.upsert_job(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar_profile_id=scholar.id,
|
||||
resume_cstart=queue_cstart,
|
||||
reason=queue_reason,
|
||||
run_id=run.id,
|
||||
delay_seconds=queue_delay_seconds,
|
||||
)
|
||||
result_entry["continuation_enqueued"] = True
|
||||
result_entry["continuation_reason"] = queue_reason
|
||||
result_entry["continuation_cstart"] = queue_cstart
|
||||
return
|
||||
if await queue_service.clear_job_for_scholar(db_session, user_id=user_id, scholar_profile_id=scholar.id):
|
||||
result_entry["continuation_cleared"] = True
|
||||
|
||||
|
||||
def resolve_continuation_queue_target(
|
||||
*,
|
||||
outcome: str,
|
||||
state: str,
|
||||
pagination_truncated_reason: str | None,
|
||||
continuation_cstart: int | None,
|
||||
fallback_cstart: int,
|
||||
) -> tuple[str | None, int]:
|
||||
if outcome == "partial":
|
||||
reason = (pagination_truncated_reason or "").strip()
|
||||
if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(RESUMABLE_PARTIAL_REASON_PREFIXES):
|
||||
return reason, queue_service.normalize_cstart(
|
||||
continuation_cstart if continuation_cstart is not None else fallback_cstart
|
||||
)
|
||||
return None, queue_service.normalize_cstart(fallback_cstart)
|
||||
|
||||
if outcome == "failed" and state == ParseState.NETWORK_ERROR.value:
|
||||
return "network_error_retry", queue_service.normalize_cstart(
|
||||
continuation_cstart if continuation_cstart is not None else fallback_cstart
|
||||
)
|
||||
|
||||
return None, queue_service.normalize_cstart(fallback_cstart)
|
||||
|
||||
|
||||
async def process_scholar(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
pagination: PaginationEngine,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
user_id: int,
|
||||
request_delay_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
rate_limit_retries: int,
|
||||
rate_limit_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
start_cstart: int,
|
||||
auto_queue_continuations: bool,
|
||||
queue_delay_seconds: int,
|
||||
) -> ScholarProcessingOutcome:
|
||||
try:
|
||||
run_dt, paged_parse_result, result_entry = await _fetch_and_prepare_scholar_result(
|
||||
db_session,
|
||||
pagination=pagination,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
user_id=user_id,
|
||||
start_cstart=start_cstart,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
rate_limit_retries=rate_limit_retries,
|
||||
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
|
||||
max_pages_per_scholar=max_pages_per_scholar,
|
||||
page_size=page_size,
|
||||
)
|
||||
outcome = await _resolve_scholar_outcome(
|
||||
db_session,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
)
|
||||
await sync_continuation_queue(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
scholar=scholar,
|
||||
run=run,
|
||||
start_cstart=start_cstart,
|
||||
result_entry=outcome.result_entry,
|
||||
paged_parse_result=paged_parse_result,
|
||||
auto_queue_continuations=auto_queue_continuations,
|
||||
queue_delay_seconds=queue_delay_seconds,
|
||||
)
|
||||
return outcome
|
||||
except Exception as exc:
|
||||
return unexpected_scholar_exception_outcome(
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
start_cstart=start_cstart,
|
||||
exc=exc,
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_and_prepare_scholar_result(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
pagination: PaginationEngine,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
user_id: int,
|
||||
start_cstart: int,
|
||||
request_delay_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
rate_limit_retries: int,
|
||||
rate_limit_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
) -> tuple[datetime, PagedParseResult, dict[str, Any]]:
|
||||
run_dt = datetime.now(UTC)
|
||||
paged_parse_result = await pagination.fetch_and_parse_all_pages(
|
||||
scholar=scholar,
|
||||
run=run,
|
||||
db_session=db_session,
|
||||
start_cstart=start_cstart,
|
||||
request_delay_seconds=request_delay_seconds,
|
||||
network_error_retries=network_error_retries,
|
||||
retry_backoff_seconds=retry_backoff_seconds,
|
||||
rate_limit_retries=rate_limit_retries,
|
||||
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
|
||||
max_pages=max_pages_per_scholar,
|
||||
page_size=page_size,
|
||||
previous_initial_page_fingerprint_sha256=scholar.last_initial_page_fingerprint_sha256,
|
||||
upsert_publications_fn=upsert_profile_publications,
|
||||
)
|
||||
assert_valid_paged_parse_result(scholar_id=scholar.scholar_id, paged_parse_result=paged_parse_result)
|
||||
apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt)
|
||||
parsed_page = paged_parse_result.parsed_page
|
||||
structured_log(
|
||||
logger,
|
||||
"info",
|
||||
"ingestion.scholar_parsed",
|
||||
user_id=user_id,
|
||||
crawl_run_id=run.id,
|
||||
scholar_profile_id=scholar.id,
|
||||
scholar_id=scholar.scholar_id,
|
||||
state=parsed_page.state.value,
|
||||
publication_count=len(paged_parse_result.publications),
|
||||
has_show_more_button=parsed_page.has_show_more_button,
|
||||
pages_fetched=paged_parse_result.pages_fetched,
|
||||
pages_attempted=paged_parse_result.pages_attempted,
|
||||
has_more_remaining=paged_parse_result.has_more_remaining,
|
||||
pagination_truncated_reason=paged_parse_result.pagination_truncated_reason,
|
||||
warning_count=len(parsed_page.warnings),
|
||||
skipped_no_change=paged_parse_result.skipped_no_change,
|
||||
)
|
||||
result_entry = build_result_entry(
|
||||
scholar=scholar,
|
||||
start_cstart=start_cstart,
|
||||
paged_parse_result=paged_parse_result,
|
||||
)
|
||||
return run_dt, paged_parse_result, result_entry
|
||||
|
||||
|
||||
async def _resolve_scholar_outcome(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
run_dt: datetime,
|
||||
paged_parse_result: PagedParseResult,
|
||||
result_entry: dict[str, Any],
|
||||
) -> ScholarProcessingOutcome:
|
||||
if paged_parse_result.skipped_no_change:
|
||||
return skipped_no_change_outcome(
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
)
|
||||
return await upsert_publications_outcome(
|
||||
db_session,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
run_dt=run_dt,
|
||||
paged_parse_result=paged_parse_result,
|
||||
result_entry=result_entry,
|
||||
)
|
||||
|
||||
|
||||
def unexpected_scholar_exception_outcome(
|
||||
*,
|
||||
run: CrawlRun,
|
||||
scholar: ScholarProfile,
|
||||
start_cstart: int,
|
||||
exc: Exception,
|
||||
) -> ScholarProcessingOutcome:
|
||||
scholar.last_run_status = RunStatus.FAILED
|
||||
scholar.last_run_dt = datetime.now(UTC)
|
||||
logger.exception(
|
||||
"ingestion.scholar_unexpected_failure",
|
||||
extra={
|
||||
"crawl_run_id": run.id,
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
},
|
||||
)
|
||||
return ScholarProcessingOutcome(
|
||||
result_entry={
|
||||
"scholar_profile_id": scholar.id,
|
||||
"scholar_id": scholar.scholar_id,
|
||||
"state": "ingestion_error",
|
||||
"state_reason": "scholar_processing_exception",
|
||||
"outcome": "failed",
|
||||
"attempt_count": 0,
|
||||
"publication_count": 0,
|
||||
"start_cstart": start_cstart,
|
||||
"warnings": [],
|
||||
"error": str(exc),
|
||||
"debug": {"exception_type": type(exc).__name__, "exception_message": str(exc)},
|
||||
},
|
||||
succeeded_count_delta=0,
|
||||
failed_count_delta=1,
|
||||
partial_count_delta=0,
|
||||
discovered_publication_count=0,
|
||||
)
|
||||
|
||||
|
||||
async def run_scholar_iteration(
|
||||
db_session: AsyncSession,
|
||||
*,
|
||||
pagination: PaginationEngine,
|
||||
run: CrawlRun,
|
||||
scholars: list[ScholarProfile],
|
||||
user_id: int,
|
||||
start_cstart_map: dict[int, int],
|
||||
request_delay_seconds: int,
|
||||
network_error_retries: int,
|
||||
retry_backoff_seconds: float,
|
||||
rate_limit_retries: int,
|
||||
rate_limit_backoff_seconds: float,
|
||||
max_pages_per_scholar: int,
|
||||
page_size: int,
|
||||
auto_queue_continuations: bool,
|
||||
queue_delay_seconds: int,
|
||||
) -> RunProgress:
|
||||
progress = RunProgress()
|
||||
scholar_kwargs = {
|
||||
"request_delay_seconds": request_delay_seconds,
|
||||
"network_error_retries": network_error_retries,
|
||||
"retry_backoff_seconds": retry_backoff_seconds,
|
||||
"rate_limit_retries": rate_limit_retries,
|
||||
"rate_limit_backoff_seconds": rate_limit_backoff_seconds,
|
||||
"page_size": page_size,
|
||||
}
|
||||
|
||||
# ── Pass 1: first page of every scholar (breadth-first) ──────────
|
||||
first_pass_cstarts: dict[int, int] = {}
|
||||
for index, scholar in enumerate(scholars):
|
||||
await db_session.refresh(run)
|
||||
if run.status == RunStatus.CANCELED:
|
||||
structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
|
||||
return progress
|
||||
if index > 0 and request_delay_seconds > 0:
|
||||
await asyncio.sleep(float(request_delay_seconds))
|
||||
start_cstart = int(start_cstart_map.get(int(scholar.id), 0))
|
||||
outcome = await process_scholar(
|
||||
db_session,
|
||||
pagination=pagination,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
user_id=user_id,
|
||||
start_cstart=start_cstart,
|
||||
max_pages_per_scholar=1,
|
||||
auto_queue_continuations=False,
|
||||
queue_delay_seconds=queue_delay_seconds,
|
||||
**scholar_kwargs,
|
||||
)
|
||||
apply_outcome_to_progress(progress=progress, outcome=outcome)
|
||||
resume_cstart = outcome.result_entry.get("continuation_cstart")
|
||||
if resume_cstart is not None and int(resume_cstart) > start_cstart:
|
||||
first_pass_cstarts[int(scholar.id)] = int(resume_cstart)
|
||||
|
||||
# ── Pass 2: remaining pages for each scholar (depth) ─────────────
|
||||
remaining_max = max(max_pages_per_scholar - 1, 0)
|
||||
if remaining_max <= 0:
|
||||
return progress
|
||||
|
||||
for index, scholar in enumerate(scholars):
|
||||
resume_cstart = first_pass_cstarts.get(int(scholar.id))
|
||||
if resume_cstart is None:
|
||||
continue
|
||||
await db_session.refresh(run)
|
||||
if run.status == RunStatus.CANCELED:
|
||||
structured_log(logger, "info", "ingestion.run_canceled", run_id=run.id, user_id=user_id)
|
||||
break
|
||||
if index > 0 and request_delay_seconds > 0:
|
||||
await asyncio.sleep(float(request_delay_seconds))
|
||||
outcome = await process_scholar(
|
||||
db_session,
|
||||
pagination=pagination,
|
||||
run=run,
|
||||
scholar=scholar,
|
||||
user_id=user_id,
|
||||
start_cstart=resume_cstart,
|
||||
max_pages_per_scholar=remaining_max,
|
||||
auto_queue_continuations=auto_queue_continuations,
|
||||
queue_delay_seconds=queue_delay_seconds,
|
||||
**scholar_kwargs,
|
||||
)
|
||||
apply_outcome_to_progress(progress=progress, outcome=outcome)
|
||||
return progress
|
||||
|
|
@ -62,6 +62,7 @@ class LiveScholarSource:
|
|||
*,
|
||||
timeout_seconds: float = 25.0,
|
||||
min_interval_seconds: float | None = None,
|
||||
rotate_user_agents: bool | None = None,
|
||||
user_agents: list[str] | None = None,
|
||||
) -> None:
|
||||
self._timeout_seconds = timeout_seconds
|
||||
|
|
@ -72,6 +73,38 @@ class LiveScholarSource:
|
|||
)
|
||||
self._min_interval_seconds = max(configured_interval, 0.0)
|
||||
self._user_agents = user_agents or DEFAULT_USER_AGENTS
|
||||
self._rotate_user_agents = (
|
||||
bool(settings.scholar_http_rotate_user_agent)
|
||||
if rotate_user_agents is None
|
||||
else bool(rotate_user_agents)
|
||||
)
|
||||
configured_user_agent = settings.scholar_http_user_agent.strip()
|
||||
self._configured_user_agent = configured_user_agent or None
|
||||
self._accept_language = settings.scholar_http_accept_language.strip() or "en-US,en;q=0.9"
|
||||
self._cookie_header = settings.scholar_http_cookie.strip() or None
|
||||
self._stable_user_agent = self._resolve_initial_user_agent()
|
||||
|
||||
def _resolve_initial_user_agent(self) -> str:
|
||||
if self._configured_user_agent is not None:
|
||||
return self._configured_user_agent
|
||||
return random.choice(self._user_agents)
|
||||
|
||||
def _resolve_user_agent_for_request(self) -> str:
|
||||
if self._configured_user_agent is not None:
|
||||
return self._configured_user_agent
|
||||
if self._rotate_user_agents:
|
||||
return random.choice(self._user_agents)
|
||||
return self._stable_user_agent
|
||||
|
||||
def _request_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": self._resolve_user_agent_for_request(),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": self._accept_language,
|
||||
}
|
||||
if self._cookie_header is not None:
|
||||
headers["Cookie"] = self._cookie_header
|
||||
return headers
|
||||
|
||||
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
|
||||
return await self.fetch_profile_page_html(
|
||||
|
|
@ -116,15 +149,21 @@ class LiveScholarSource:
|
|||
return await asyncio.to_thread(self._fetch_sync, requested_url)
|
||||
|
||||
def _build_request(self, requested_url: str) -> Request:
|
||||
return Request(
|
||||
requested_url,
|
||||
headers={
|
||||
"User-Agent": random.choice(self._user_agents),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
return Request(requested_url, headers=self._request_headers())
|
||||
|
||||
@staticmethod
|
||||
def _http_error_reason(*, status_code: int, final_url: str, body: str) -> str:
|
||||
lowered_url = final_url.lower()
|
||||
lowered_body = body.lower()
|
||||
if "sorry/index" in lowered_url or "sorry/index" in lowered_body:
|
||||
return "blocked_google_sorry_challenge"
|
||||
if "our systems have detected" in lowered_body or "unusual traffic" in lowered_body:
|
||||
return "blocked_unusual_traffic_detected"
|
||||
if "automated queries" in lowered_body:
|
||||
return "blocked_automated_queries_detected"
|
||||
if status_code == 429:
|
||||
return "blocked_http_429_rate_limited"
|
||||
return f"http_error_status_{status_code}"
|
||||
|
||||
@staticmethod
|
||||
def _http_error_body(exc: HTTPError) -> str:
|
||||
|
|
@ -151,18 +190,27 @@ class LiveScholarSource:
|
|||
|
||||
@staticmethod
|
||||
def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
|
||||
final_url = exc.geturl()
|
||||
body = LiveScholarSource._http_error_body(exc)
|
||||
block_reason = LiveScholarSource._http_error_reason(
|
||||
status_code=exc.code,
|
||||
final_url=final_url,
|
||||
body=body,
|
||||
)
|
||||
structured_log(
|
||||
logger,
|
||||
"warning",
|
||||
"scholar_source.fetch_http_error",
|
||||
requested_url=requested_url,
|
||||
status_code=exc.code,
|
||||
final_url=final_url,
|
||||
block_reason=block_reason,
|
||||
)
|
||||
return FetchResult(
|
||||
requested_url=requested_url,
|
||||
status_code=exc.code,
|
||||
final_url=exc.geturl(),
|
||||
body=LiveScholarSource._http_error_body(exc),
|
||||
final_url=final_url,
|
||||
body=body,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,18 +38,18 @@ def classify_block_or_captcha_reason(
|
|||
) -> str | None:
|
||||
if "accounts.google.com" in final_url and ("signin" in final_url or "servicelogin" in final_url):
|
||||
return "blocked_accounts_redirect"
|
||||
if status_code == 429:
|
||||
return "blocked_http_429_rate_limited"
|
||||
if status_code == 403:
|
||||
if "recaptcha" in body_lowered or "captcha" in body_lowered or "sorry/index" in final_url:
|
||||
return "blocked_http_403_captcha_challenge"
|
||||
return "blocked_http_403_forbidden"
|
||||
if "sorry/index" in final_url or "sorry/index" in body_lowered:
|
||||
return "blocked_google_sorry_challenge"
|
||||
if "our systems have detected" in body_lowered or "unusual traffic" in body_lowered:
|
||||
return "blocked_unusual_traffic_detected"
|
||||
if "automated queries" in body_lowered:
|
||||
return "blocked_automated_queries_detected"
|
||||
if status_code == 429:
|
||||
return "blocked_http_429_rate_limited"
|
||||
if status_code == 403:
|
||||
if "recaptcha" in body_lowered or "captcha" in body_lowered:
|
||||
return "blocked_http_403_captcha_challenge"
|
||||
return "blocked_http_403_forbidden"
|
||||
if "not a robot" in body_lowered:
|
||||
return "blocked_not_a_robot_challenge"
|
||||
if "recaptcha" in body_lowered:
|
||||
|
|
|
|||
|
|
@ -235,6 +235,13 @@ class Settings:
|
|||
"SCHOLAR_NAME_SEARCH_ALERT_COOLDOWN_REJECTIONS_THRESHOLD",
|
||||
3,
|
||||
)
|
||||
scholar_http_user_agent: str = _env_str("SCHOLAR_HTTP_USER_AGENT", "")
|
||||
scholar_http_rotate_user_agent: bool = _env_bool("SCHOLAR_HTTP_ROTATE_USER_AGENT", False)
|
||||
scholar_http_accept_language: str = _env_str(
|
||||
"SCHOLAR_HTTP_ACCEPT_LANGUAGE",
|
||||
"en-US,en;q=0.9",
|
||||
)
|
||||
scholar_http_cookie: str = _env_str("SCHOLAR_HTTP_COOKIE", "")
|
||||
unpaywall_enabled: bool = _env_bool("UNPAYWALL_ENABLED", True)
|
||||
unpaywall_email: str = _env_str("UNPAYWALL_EMAIL", "")
|
||||
unpaywall_timeout_seconds: float = _env_float("UNPAYWALL_TIMEOUT_SECONDS", 4.0)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,13 @@ export interface UserSettingsUpdate {
|
|||
crossref_api_mailto: string | null;
|
||||
}
|
||||
|
||||
export interface AdminScholarHttpSettings {
|
||||
user_agent: string;
|
||||
rotate_user_agent: boolean;
|
||||
accept_language: string;
|
||||
cookie: string;
|
||||
}
|
||||
|
||||
export interface ChangePasswordPayload {
|
||||
current_password: string;
|
||||
new_password: string;
|
||||
|
|
@ -60,3 +67,20 @@ export async function changePassword(payload: ChangePasswordPayload): Promise<{
|
|||
});
|
||||
return response.data;
|
||||
}
|
||||
|
||||
export async function fetchAdminScholarHttpSettings(): Promise<AdminScholarHttpSettings> {
|
||||
const response = await apiRequest<AdminScholarHttpSettings>("/admin/settings/scholar-http", {
|
||||
method: "GET",
|
||||
});
|
||||
return response.data;
|
||||
}
|
||||
|
||||
export async function updateAdminScholarHttpSettings(
|
||||
payload: AdminScholarHttpSettings,
|
||||
): Promise<AdminScholarHttpSettings> {
|
||||
const response = await apiRequest<AdminScholarHttpSettings>("/admin/settings/scholar-http", {
|
||||
method: "PUT",
|
||||
body: payload,
|
||||
});
|
||||
return response.data;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,9 +14,12 @@ import AppTabs, { type AppTabItem } from "@/components/ui/AppTabs.vue";
|
|||
import SettingsAdminPanel from "@/features/settings/SettingsAdminPanel.vue";
|
||||
import {
|
||||
changePassword,
|
||||
fetchAdminScholarHttpSettings,
|
||||
fetchSettings,
|
||||
type AdminScholarHttpSettings,
|
||||
type UserSettings,
|
||||
type UserSettingsUpdate,
|
||||
updateAdminScholarHttpSettings,
|
||||
updateSettings,
|
||||
} from "@/features/settings";
|
||||
import { ApiRequestError } from "@/lib/api/errors";
|
||||
|
|
@ -40,6 +43,7 @@ const router = useRouter();
|
|||
|
||||
const loading = ref(true);
|
||||
const saving = ref(false);
|
||||
const savingScholarHttp = ref(false);
|
||||
const updatingPassword = ref(false);
|
||||
|
||||
const autoRunEnabled = ref(false);
|
||||
|
|
@ -49,6 +53,10 @@ const navVisiblePages = ref<string[]>([]);
|
|||
const openalexApiKey = ref("");
|
||||
const crossrefApiToken = ref("");
|
||||
const crossrefApiMailto = ref("");
|
||||
const scholarHttpUserAgent = ref("");
|
||||
const scholarHttpRotateUserAgent = ref(false);
|
||||
const scholarHttpAcceptLanguage = ref("en-US,en;q=0.9");
|
||||
const scholarHttpCookie = ref("");
|
||||
|
||||
const currentPassword = ref("");
|
||||
const newPassword = ref("");
|
||||
|
|
@ -142,6 +150,13 @@ function hydrateSettings(settings: UserSettings): void {
|
|||
runStatus.setSafetyState(settings.safety_state);
|
||||
}
|
||||
|
||||
function hydrateScholarHttpSettings(settings: AdminScholarHttpSettings): void {
|
||||
scholarHttpUserAgent.value = settings.user_agent;
|
||||
scholarHttpRotateUserAgent.value = Boolean(settings.rotate_user_agent);
|
||||
scholarHttpAcceptLanguage.value = settings.accept_language;
|
||||
scholarHttpCookie.value = settings.cookie;
|
||||
}
|
||||
|
||||
function parseBoundedInteger(value: string, label: string, minimum: number): number {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed)) {
|
||||
|
|
@ -160,6 +175,9 @@ async function loadSettings(): Promise<void> {
|
|||
try {
|
||||
const settings = await fetchSettings();
|
||||
hydrateSettings(settings);
|
||||
if (auth.isAdmin) {
|
||||
hydrateScholarHttpSettings(await fetchAdminScholarHttpSettings());
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ApiRequestError) {
|
||||
errorMessage.value = error.message;
|
||||
|
|
@ -172,6 +190,32 @@ async function loadSettings(): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
async function onSaveScholarHttpSettings(): Promise<void> {
|
||||
savingScholarHttp.value = true;
|
||||
errorMessage.value = null;
|
||||
errorRequestId.value = null;
|
||||
successMessage.value = null;
|
||||
try {
|
||||
const updated = await updateAdminScholarHttpSettings({
|
||||
user_agent: scholarHttpUserAgent.value.trim(),
|
||||
rotate_user_agent: scholarHttpRotateUserAgent.value,
|
||||
accept_language: scholarHttpAcceptLanguage.value.trim(),
|
||||
cookie: scholarHttpCookie.value.trim(),
|
||||
});
|
||||
hydrateScholarHttpSettings(updated);
|
||||
successMessage.value = "Scholar HTTP profile updated.";
|
||||
} catch (error) {
|
||||
if (error instanceof ApiRequestError) {
|
||||
errorMessage.value = error.message;
|
||||
errorRequestId.value = error.requestId;
|
||||
} else {
|
||||
errorMessage.value = "Unable to save Scholar HTTP profile.";
|
||||
}
|
||||
} finally {
|
||||
savingScholarHttp.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function onSaveSettings(): Promise<void> {
|
||||
saving.value = true;
|
||||
errorMessage.value = null;
|
||||
|
|
@ -404,6 +448,49 @@ onMounted(async () => {
|
|||
<AppInput v-model="crossrefApiMailto" placeholder="e.g. admin@yourdomain.com" type="email" autocomplete="off" />
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="grid gap-2 rounded-lg border border-stroke-default bg-surface-card-muted p-3">
|
||||
<h3 class="text-sm font-semibold text-ink-secondary">Scholar Request Profile</h3>
|
||||
<p class="text-xs text-secondary mb-2">
|
||||
Tune Scholar HTTP fingerprint behavior used by live scraper requests. Changes apply to new runs.
|
||||
</p>
|
||||
<label class="grid gap-1 text-sm font-medium text-ink-secondary">
|
||||
<span>User-Agent override</span>
|
||||
<AppInput
|
||||
v-model="scholarHttpUserAgent"
|
||||
placeholder="Leave empty to use built-in browser-like user agent"
|
||||
autocomplete="off"
|
||||
/>
|
||||
</label>
|
||||
<label class="grid gap-1 text-sm font-medium text-ink-secondary mt-2">
|
||||
<span>Accept-Language</span>
|
||||
<AppInput
|
||||
v-model="scholarHttpAcceptLanguage"
|
||||
placeholder="e.g. en-US,en;q=0.9"
|
||||
autocomplete="off"
|
||||
/>
|
||||
</label>
|
||||
<label class="grid gap-1 text-sm font-medium text-ink-secondary mt-2">
|
||||
<span>Cookie header</span>
|
||||
<AppInput
|
||||
v-model="scholarHttpCookie"
|
||||
placeholder="Optional. Leave empty to disable cookie passthrough"
|
||||
autocomplete="off"
|
||||
/>
|
||||
</label>
|
||||
<div class="mt-2">
|
||||
<AppCheckbox
|
||||
id="scholar-http-rotate-user-agent"
|
||||
v-model="scholarHttpRotateUserAgent"
|
||||
label="Rotate user-agent per request"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<AppButton :disabled="savingScholarHttp" @click="onSaveScholarHttpSettings">
|
||||
{{ savingScholarHttp ? "Saving..." : "Save Scholar request profile" }}
|
||||
</AppButton>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -391,6 +391,61 @@ async def test_api_admin_user_management_endpoints(db_session: AsyncSession) ->
|
|||
assert created_exists.scalar_one() == 1
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_admin_scholar_http_settings_endpoints(db_session: AsyncSession) -> None:
|
||||
await insert_user(
|
||||
db_session,
|
||||
email="api-admin-http@example.com",
|
||||
password="admin-password",
|
||||
is_admin=True,
|
||||
)
|
||||
await insert_user(
|
||||
db_session,
|
||||
email="api-member-http@example.com",
|
||||
password="member-password",
|
||||
is_admin=False,
|
||||
)
|
||||
previous_user_agent = settings.scholar_http_user_agent
|
||||
previous_rotate = settings.scholar_http_rotate_user_agent
|
||||
previous_accept_language = settings.scholar_http_accept_language
|
||||
previous_cookie = settings.scholar_http_cookie
|
||||
try:
|
||||
client = TestClient(app)
|
||||
login_user(client, email="api-admin-http@example.com", password="admin-password")
|
||||
headers = _api_csrf_headers(client)
|
||||
|
||||
read_response = client.get("/api/v1/admin/settings/scholar-http")
|
||||
assert read_response.status_code == 200
|
||||
|
||||
update_response = client.put(
|
||||
"/api/v1/admin/settings/scholar-http",
|
||||
json={
|
||||
"user_agent": "Mozilla/5.0 Test Runner",
|
||||
"rotate_user_agent": False,
|
||||
"accept_language": "en-US,en;q=0.8",
|
||||
"cookie": "SID=test-cookie",
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
assert update_response.status_code == 200
|
||||
payload = update_response.json()["data"]
|
||||
assert payload["user_agent"] == "Mozilla/5.0 Test Runner"
|
||||
assert payload["cookie"] == "SID=test-cookie"
|
||||
assert settings.scholar_http_user_agent == "Mozilla/5.0 Test Runner"
|
||||
|
||||
client.post("/api/v1/auth/logout", headers=headers)
|
||||
login_user(client, email="api-member-http@example.com", password="member-password")
|
||||
forbidden_response = client.get("/api/v1/admin/settings/scholar-http")
|
||||
assert forbidden_response.status_code == 403
|
||||
finally:
|
||||
object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
|
||||
object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
|
||||
object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
|
||||
object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.db
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from sqlalchemy import select
|
|||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import CrawlRun, Publication, RunStatus, RunTriggerType, ScholarProfile, ScholarPublication
|
||||
from app.services.ingestion.application import ScholarIngestionService
|
||||
from app.services.ingestion.enrichment import EnrichmentRunner
|
||||
from app.services.openalex.types import OpenAlexWork
|
||||
from tests.integration.helpers import insert_user
|
||||
|
||||
|
|
@ -79,8 +79,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
|
|||
oa_url="http://example.com/grover.pdf",
|
||||
)
|
||||
|
||||
mock_source = MagicMock()
|
||||
service = ScholarIngestionService(source=mock_source)
|
||||
runner = EnrichmentRunner()
|
||||
|
||||
# We patch the client at its source, and also mock arXiv to avoid real HTTP calls
|
||||
with (
|
||||
|
|
@ -93,7 +92,7 @@ async def test_deferred_enrichment_sweeps_previous_runs(db_session: AsyncSession
|
|||
mock_instance.get_works_by_filter = AsyncMock(return_value=[mock_work])
|
||||
|
||||
# 5. Execute the enrichment pass for the NEW run
|
||||
await service._enrich_pending_publications(db_session, run_id=new_run.id)
|
||||
await runner.enrich_pending_publications(db_session, run_id=new_run.id)
|
||||
await db_session.commit()
|
||||
|
||||
# 6. Verification: The publication from the FAILED run should now be enriched
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from types import SimpleNamespace
|
|||
import pytest
|
||||
|
||||
from app.services.arxiv.errors import ArxivRateLimitError
|
||||
from app.services.ingestion.application import ScholarIngestionService
|
||||
from app.services.ingestion.enrichment import EnrichmentRunner
|
||||
from app.services.publication_identifiers import application as identifier_service
|
||||
|
||||
|
||||
|
|
@ -13,7 +13,7 @@ from app.services.publication_identifiers import application as identifier_servi
|
|||
async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
service = ScholarIngestionService(source=object())
|
||||
runner = EnrichmentRunner()
|
||||
publication = SimpleNamespace(id=11, author_text="Ada Lovelace")
|
||||
calls = {"sync": 0}
|
||||
|
||||
|
|
@ -35,9 +35,9 @@ async def test_discover_identifiers_for_enrichment_disables_arxiv_on_rate_limit(
|
|||
async def _publish_noop(*args, **kwargs) -> None:
|
||||
_ = (args, kwargs)
|
||||
|
||||
monkeypatch.setattr(service, "_publish_identifier_update_event", _publish_noop)
|
||||
monkeypatch.setattr(runner, "_publish_identifier_update_event", _publish_noop)
|
||||
|
||||
result = await service._discover_identifiers_for_enrichment(
|
||||
result = await runner._discover_identifiers_for_enrichment(
|
||||
object(),
|
||||
publication=publication,
|
||||
run_id=321,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import pytest
|
||||
|
||||
from app.services.ingestion.application import ScholarIngestionService
|
||||
from app.services.ingestion.run_completion import apply_outcome_to_progress
|
||||
from app.services.ingestion.types import RunProgress, ScholarProcessingOutcome
|
||||
|
||||
|
||||
|
|
@ -31,11 +31,11 @@ def _outcome(*, scholar_profile_id: int, outcome_label: str) -> ScholarProcessin
|
|||
def test_apply_outcome_to_progress_replaces_previous_scholar_outcome() -> None:
|
||||
progress = RunProgress()
|
||||
|
||||
ScholarIngestionService._apply_outcome_to_progress(
|
||||
apply_outcome_to_progress(
|
||||
progress=progress,
|
||||
outcome=_outcome(scholar_profile_id=42, outcome_label="partial"),
|
||||
)
|
||||
ScholarIngestionService._apply_outcome_to_progress(
|
||||
apply_outcome_to_progress(
|
||||
progress=progress,
|
||||
outcome=_outcome(scholar_profile_id=42, outcome_label="success"),
|
||||
)
|
||||
|
|
@ -58,4 +58,4 @@ def test_apply_outcome_to_progress_rejects_invalid_scholar_id() -> None:
|
|||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="missing valid scholar_profile_id"):
|
||||
ScholarIngestionService._apply_outcome_to_progress(progress=progress, outcome=invalid)
|
||||
apply_outcome_to_progress(progress=progress, outcome=invalid)
|
||||
|
|
|
|||
|
|
@ -433,3 +433,18 @@ def test_parse_author_search_page_classifies_http_429_as_blocked() -> None:
|
|||
|
||||
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
|
||||
assert parsed.state_reason == "blocked_http_429_rate_limited"
|
||||
|
||||
|
||||
def test_parse_author_search_page_prefers_sorry_challenge_reason_over_generic_429() -> None:
|
||||
fetch_result = FetchResult(
|
||||
requested_url="https://scholar.google.com/citations?hl=en&view_op=search_authors&mauthors=ada",
|
||||
status_code=429,
|
||||
final_url="https://www.google.com/sorry/index?continue=scholar",
|
||||
body="<html><body>Our systems have detected unusual traffic.</body></html>",
|
||||
error=None,
|
||||
)
|
||||
|
||||
parsed = parse_author_search_page(fetch_result)
|
||||
|
||||
assert parsed.state == ParseState.BLOCKED_OR_CAPTCHA
|
||||
assert parsed.state_reason == "blocked_google_sorry_challenge"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,12 @@ import pytest
|
|||
|
||||
from app.services.scholar import rate_limit as scholar_rate_limit
|
||||
from app.services.scholar.source import FetchResult, LiveScholarSource, _build_profile_url
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
def _request_header(request, name: str) -> str | None:
|
||||
headers = {key.lower(): value for key, value in request.header_items()}
|
||||
return headers.get(name.lower())
|
||||
|
||||
|
||||
def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
|
||||
|
|
@ -48,3 +54,72 @@ async def test_live_scholar_source_applies_global_throttle(monkeypatch: pytest.M
|
|||
|
||||
assert result == expected_result
|
||||
assert captured_interval == [7.0]
|
||||
|
||||
|
||||
def test_http_error_reason_prefers_sorry_challenge_marker() -> None:
|
||||
reason = LiveScholarSource._http_error_reason(
|
||||
status_code=429,
|
||||
final_url="https://www.google.com/sorry/index?continue=example",
|
||||
body="",
|
||||
)
|
||||
assert reason == "blocked_google_sorry_challenge"
|
||||
|
||||
|
||||
def test_http_error_reason_falls_back_to_http_429_rate_limit() -> None:
|
||||
reason = LiveScholarSource._http_error_reason(
|
||||
status_code=429,
|
||||
final_url="https://scholar.google.com/citations?hl=en&user=abc",
|
||||
body="Too Many Requests",
|
||||
)
|
||||
assert reason == "blocked_http_429_rate_limited"
|
||||
|
||||
|
||||
def test_build_request_uses_stable_user_agent_by_default() -> None:
|
||||
previous_user_agent = settings.scholar_http_user_agent
|
||||
previous_rotate = settings.scholar_http_rotate_user_agent
|
||||
previous_cookie = settings.scholar_http_cookie
|
||||
previous_accept_language = settings.scholar_http_accept_language
|
||||
object.__setattr__(settings, "scholar_http_user_agent", "")
|
||||
object.__setattr__(settings, "scholar_http_rotate_user_agent", False)
|
||||
object.__setattr__(settings, "scholar_http_cookie", "")
|
||||
object.__setattr__(settings, "scholar_http_accept_language", "en-US,en;q=0.9")
|
||||
try:
|
||||
source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
|
||||
request_one = source._build_request("https://example.test/one")
|
||||
request_two = source._build_request("https://example.test/two")
|
||||
assert _request_header(request_one, "User-Agent") == _request_header(request_two, "User-Agent")
|
||||
assert _request_header(request_one, "Connection") is None
|
||||
finally:
|
||||
object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
|
||||
object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
|
||||
object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
|
||||
object.__setattr__(settings, "scholar_http_accept_language", previous_accept_language)
|
||||
|
||||
|
||||
def test_build_request_rotates_user_agent_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
previous_user_agent = settings.scholar_http_user_agent
|
||||
previous_rotate = settings.scholar_http_rotate_user_agent
|
||||
object.__setattr__(settings, "scholar_http_user_agent", "")
|
||||
object.__setattr__(settings, "scholar_http_rotate_user_agent", True)
|
||||
choices = iter(["UA-STABLE", "UA-1", "UA-2"])
|
||||
monkeypatch.setattr("app.services.scholar.source.random.choice", lambda _values: next(choices))
|
||||
try:
|
||||
source = LiveScholarSource(user_agents=["UA-A", "UA-B"])
|
||||
request_one = source._build_request("https://example.test/one")
|
||||
request_two = source._build_request("https://example.test/two")
|
||||
assert _request_header(request_one, "User-Agent") == "UA-1"
|
||||
assert _request_header(request_two, "User-Agent") == "UA-2"
|
||||
finally:
|
||||
object.__setattr__(settings, "scholar_http_user_agent", previous_user_agent)
|
||||
object.__setattr__(settings, "scholar_http_rotate_user_agent", previous_rotate)
|
||||
|
||||
|
||||
def test_build_request_includes_cookie_when_configured() -> None:
|
||||
previous_cookie = settings.scholar_http_cookie
|
||||
object.__setattr__(settings, "scholar_http_cookie", "SID=abc123")
|
||||
try:
|
||||
source = LiveScholarSource(user_agents=["UA-A"])
|
||||
request = source._build_request("https://example.test/one")
|
||||
assert _request_header(request, "Cookie") == "SID=abc123"
|
||||
finally:
|
||||
object.__setattr__(settings, "scholar_http_cookie", previous_cookie)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue