justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

ci: add ruff linting and mypy type checking

Browse source 
Add ruff and mypy to dev dependencies with configuration in pyproject.toml.
Add a lint CI job that runs ruff check, ruff format --check, and mypy.
Auto-fix import sorting and formatting across the codebase. Exclude
alembic/versions from linting (auto-generated migrations). Ignore B008
(FastAPI Depends pattern) and RUF001 (unicode in user-facing strings).

21 ruff lint errors and 50 mypy errors remain for manual review.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-26 22:11:41 +01:00
parent 399276ea69
commit bf04c77aa9
137 changed files with 3066 additions and 1900 deletions
Download patch file Download diff file Expand all files Collapse all files

1
app/services/__init__.py
View file

@ -1,2 +1 @@
"""Service layer for scholarr application workflows."""

2
app/services/domains/arxiv/application.py
View file

@ -1,11 +1,11 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from app.services.domains.arxiv.gateway import (
build_arxiv_query,
get_arxiv_gateway,
)
from app.services.domains.arxiv.errors import ArxivRateLimitError
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem

64
app/services/domains/arxiv/cache.py
View file

@ -1,11 +1,11 @@
from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import asdict
from datetime import datetime, timedelta, timezone
import hashlib
import json
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import asdict
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, select
@ -33,15 +33,12 @@ async def get_cached_feed(
) -> ArxivFeed | None:
timestamp = _as_utc(now_utc)
session_factory = get_session_factory()
async with session_factory() as db_session:
async with db_session.begin():
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
)
)
entry = result.scalar_one_or_none()
return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
async with session_factory() as db_session, db_session.begin():
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
)
entry = result.scalar_one_or_none()
return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
async def set_cached_feed(
@ -54,16 +51,15 @@ async def set_cached_feed(
) -> None:
timestamp = _as_utc(now_utc)
session_factory = get_session_factory()
async with session_factory() as db_session:
async with db_session.begin():
await _write_cached_entry(
db_session,
query_fingerprint=query_fingerprint,
feed=feed,
ttl_seconds=ttl_seconds,
max_entries=max_entries,
now_utc=timestamp,
)
async with session_factory() as db_session, db_session.begin():
await _write_cached_entry(
db_session,
query_fingerprint=query_fingerprint,
feed=feed,
ttl_seconds=ttl_seconds,
max_entries=max_entries,
now_utc=timestamp,
)
async def run_with_inflight_dedupe(
@ -142,9 +138,7 @@ async def _write_cached_entry(
) -> None:
ttl = max(float(ttl_seconds), 0.0)
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
)
select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
)
existing = result.scalar_one_or_none()
if ttl <= 0.0:
@ -177,30 +171,22 @@ async def _prune_cache_entries(
now_utc: datetime,
max_entries: int,
) -> None:
await db_session.execute(
delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc)
)
await db_session.execute(delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc))
bounded_max_entries = int(max_entries)
if bounded_max_entries <= 0:
return
count_result = await db_session.execute(
select(func.count()).select_from(ArxivQueryCacheEntry)
)
count_result = await db_session.execute(select(func.count()).select_from(ArxivQueryCacheEntry))
entry_count = int(count_result.scalar_one() or 0)
overflow = max(0, entry_count - bounded_max_entries)
if overflow <= 0:
return
stale_result = await db_session.execute(
select(ArxivQueryCacheEntry.query_fingerprint)
.order_by(ArxivQueryCacheEntry.cached_at.asc())
.limit(overflow)
select(ArxivQueryCacheEntry.query_fingerprint).order_by(ArxivQueryCacheEntry.cached_at.asc()).limit(overflow)
)
stale_keys = [str(row[0]) for row in stale_result.all()]
if stale_keys:
await db_session.execute(
delete(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys)
)
delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys))
)
@ -275,9 +261,9 @@ def _as_string_list(value: object) -> list[str]:
def _as_utc(value: datetime | None) -> datetime:
if value is None:
return datetime.now(timezone.utc)
return datetime.now(UTC)
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.replace(tzinfo=UTC)
return value

20
app/services/domains/arxiv/client.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
import logging
from collections.abc import Awaitable, Callable
import httpx
@ -103,9 +103,13 @@ class ArxivClient:
if self._cache_enabled:
cached = await get_cached_feed(query_fingerprint=query_fingerprint)
if cached is not None:
structured_log(logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path)
structured_log(
logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path
)
return cached
structured_log(logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path)
structured_log(
logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path
)
return await run_with_inflight_dedupe(
query_fingerprint=query_fingerprint,
fetch_feed=lambda: self._fetch_live_feed(
@ -216,13 +220,13 @@ async def _request_arxiv_feed(
cooldown_status = await get_arxiv_cooldown_status()
if cooldown_status.is_active:
structured_log(
logger, "warning", "arxiv.request_skipped_cooldown",
logger,
"warning",
"arxiv.request_skipped_cooldown",
source_path=source_path,
cooldown_remaining_seconds=float(cooldown_status.remaining_seconds),
)
raise ArxivRateLimitError(
f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)"
)
raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)")
async def _fetch() -> httpx.Response:
timeout_value = _timeout_seconds(timeout_seconds)
@ -272,5 +276,3 @@ def _source_path_from_params(params: dict[str, object]) -> str:
if "id_list" in params:
return ARXIV_SOURCE_PATH_LOOKUP_IDS
return ARXIV_SOURCE_PATH_UNKNOWN

2
app/services/domains/arxiv/gateway.py
View file

@ -2,8 +2,8 @@ from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING, Protocol
import unicodedata
from typing import TYPE_CHECKING, Protocol
from app.logging_utils import structured_log
from app.services.domains.arxiv.client import ArxivClient

52
app/services/domains/arxiv/rate_limit.py
View file

@ -1,10 +1,10 @@
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
import logging
from datetime import UTC, datetime, timedelta
import httpx
from sqlalchemy import select, text
@ -47,13 +47,11 @@ async def run_with_global_arxiv_limit(
async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> ArxivCooldownStatus:
timestamp = _normalize_datetime(now_utc) or datetime.now(timezone.utc)
timestamp = _normalize_datetime(now_utc) or datetime.now(UTC)
session_factory = get_session_factory()
async with session_factory() as db_session:
result = await db_session.execute(
select(ArxivRuntimeState.cooldown_until).where(
ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY
)
select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
)
cooldown_until = _normalize_datetime(result.scalar_one_or_none())
remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
@ -85,10 +83,14 @@ async def _run_serialized_fetch(
source_path=source_path,
)
structured_log(
logger, "info", "arxiv.request_completed",
logger,
"info",
"arxiv.request_completed",
status_code=int(response.status_code),
wait_seconds=wait_seconds,
cooldown_remaining_seconds=_cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=datetime.now(timezone.utc)),
cooldown_remaining_seconds=_cooldown_remaining_seconds(
runtime_state.cooldown_until, now_utc=datetime.now(UTC)
),
source_path=source_path,
)
return response, hit_rate_limit
@ -106,9 +108,7 @@ async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
async def _load_runtime_state_for_update(db_session: AsyncSession) -> ArxivRuntimeState:
result = await db_session.execute(
select(ArxivRuntimeState)
.where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
.with_for_update()
select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY).with_for_update()
)
state = result.scalar_one_or_none()
if state is not None:
@ -124,13 +124,27 @@ async def _wait_for_allowed_slot_or_raise(
*,
source_path: str,
) -> float:
now_utc = datetime.now(timezone.utc)
now_utc = datetime.now(UTC)
cooldown_seconds = _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc)
if cooldown_seconds > 0:
structured_log(logger, "info", "arxiv.request_scheduled", wait_seconds=0.0, source_path=source_path, cooldown_remaining_seconds=cooldown_seconds)
structured_log(
logger,
"info",
"arxiv.request_scheduled",
wait_seconds=0.0,
source_path=source_path,
cooldown_remaining_seconds=cooldown_seconds,
)
raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_seconds:.0f}s remaining)")
wait_seconds = _next_allowed_wait_seconds(runtime_state.next_allowed_at, now_utc=now_utc)
structured_log(logger, "info", "arxiv.request_scheduled", wait_seconds=wait_seconds, source_path=source_path, cooldown_remaining_seconds=0.0)
structured_log(
logger,
"info",
"arxiv.request_scheduled",
wait_seconds=wait_seconds,
source_path=source_path,
cooldown_remaining_seconds=0.0,
)
if wait_seconds > 0:
await asyncio.sleep(wait_seconds)
return wait_seconds
@ -142,13 +156,15 @@ def _record_post_response_state(
response_status: int,
source_path: str,
) -> bool:
now_utc = datetime.now(timezone.utc)
now_utc = datetime.now(UTC)
runtime_state.next_allowed_at = now_utc + timedelta(seconds=_min_interval_seconds())
if response_status == 429:
cooldown_seconds = _cooldown_seconds()
runtime_state.cooldown_until = now_utc + timedelta(seconds=cooldown_seconds)
structured_log(
logger, "warning", "arxiv.cooldown_activated",
logger,
"warning",
"arxiv.cooldown_activated",
cooldown_remaining_seconds=cooldown_seconds,
source_path=source_path,
)
@ -176,7 +192,7 @@ def _normalize_datetime(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.replace(tzinfo=UTC)
return value
@ -186,5 +202,3 @@ def _min_interval_seconds() -> float:
def _cooldown_seconds() -> float:
return max(float(settings.arxiv_rate_limit_cooldown_seconds), 0.0)

4
app/services/domains/crossref/application.py
View file

@ -9,8 +9,8 @@ from typing import TYPE_CHECKING
from crossref.restful import Etiquette, Works
from app.services.domains.doi.normalize import normalize_doi
from app.logging_utils import structured_log
from app.services.domains.doi.normalize import normalize_doi
from app.settings import settings
if TYPE_CHECKING:
@ -38,7 +38,7 @@ def _rate_limit_wait(min_interval_seconds: float) -> None:
def _normalized_tokens(value: str) -> list[str]:
lowered = value.lower().replace("’", "'").replace("“", "\"").replace("”", "\"")
lowered = value.lower().replace("’", "'").replace("“", '"').replace("”", '"')
lowered = NON_ALNUM_RE.sub(" ", lowered)
return [token for token in TOKEN_RE.findall(lowered) if len(token) >= 3]

2
app/services/domains/dbops/__init__.py
View file

@ -1,8 +1,8 @@
from app.services.domains.dbops.application import run_publication_link_repair
from app.services.domains.dbops.integrity import collect_integrity_report
from app.services.domains.dbops.near_duplicate_repair import (
run_publication_near_duplicate_repair,
)
from app.services.domains.dbops.integrity import collect_integrity_report
from app.services.domains.dbops.query import list_repair_jobs
__all__ = [

23
app/services/domains/dbops/application.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, exists, func, select, update
@ -8,7 +8,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import DataRepairJob, IngestionQueueItem, Publication, ScholarProfile, ScholarPublication
REPAIR_STATUS_PLANNED = "planned"
REPAIR_STATUS_RUNNING = "running"
REPAIR_STATUS_COMPLETED = "completed"
@ -18,7 +17,7 @@ SCOPE_MODE_ALL_USERS = "all_users"
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
return datetime.now(UTC)
def _normalize_scope_mode(scope_mode: str) -> str:
@ -105,11 +104,7 @@ async def _count_orphan_publications(db_session: AsyncSession) -> int:
stmt = (
select(func.count())
.select_from(Publication)
.where(
~exists(
select(1).where(ScholarPublication.publication_id == Publication.id)
)
)
.where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
)
result = await db_session.execute(stmt)
return int(result.scalar_one() or 0)
@ -158,9 +153,7 @@ def _job_summary(
async def _delete_links_for_targets(db_session: AsyncSession, *, target_scholar_profile_ids: list[int]) -> int:
result = await db_session.execute(
delete(ScholarPublication).where(
ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids)
)
delete(ScholarPublication).where(ScholarPublication.scholar_profile_id.in_(target_scholar_profile_ids))
)
return int(result.rowcount or 0)
@ -171,9 +164,7 @@ async def _delete_queue_for_targets(
user_id: int | None,
target_scholar_profile_ids: list[int],
) -> int:
stmt = delete(IngestionQueueItem).where(
IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids)
)
stmt = delete(IngestionQueueItem).where(IngestionQueueItem.scholar_profile_id.in_(target_scholar_profile_ids))
if user_id is not None:
stmt = stmt.where(IngestionQueueItem.user_id == user_id)
result = await db_session.execute(stmt)
@ -203,9 +194,7 @@ async def _reset_scholar_tracking_state(
async def _delete_orphan_publications(db_session: AsyncSession) -> int:
result = await db_session.execute(
delete(Publication).where(
~exists(select(1).where(ScholarPublication.publication_id == Publication.id))
)
delete(Publication).where(~exists(select(1).where(ScholarPublication.publication_id == Publication.id)))
)
return int(result.rowcount or 0)

4
app/services/domains/dbops/integrity.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, text
@ -168,7 +168,7 @@ async def collect_integrity_report(db_session: AsyncSession) -> dict[str, Any]:
return {
"status": _status_from_issues(failures=failures, warnings=warnings),
"checked_at": datetime.now(timezone.utc).isoformat(),
"checked_at": datetime.now(UTC).isoformat(),
"failures": failures,
"warnings": warnings,
"checks": checks,

4
app/services/domains/dbops/near_duplicate_repair.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
@ -17,7 +17,7 @@ NEAR_DUP_DEFAULT_MAX_CLUSTERS = 25
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
return datetime.now(UTC)
def _normalized_cluster_keys(values: list[str] | None) -> list[str]:

4
app/services/domains/dbops/query.py
View file

@ -16,7 +16,5 @@ async def list_repair_jobs(
limit: int = 50,
) -> list[DataRepairJob]:
bounded = _bounded_limit(limit)
result = await db_session.execute(
select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded)
)
result = await db_session.execute(select(DataRepairJob).order_by(DataRepairJob.created_at.desc()).limit(bounded))
return list(result.scalars())

372
app/services/domains/ingestion/application.py
View file

@ -1,10 +1,10 @@
from __future__ import annotations
import asyncio
from datetime import datetime, timedelta, timezone
import hashlib
import logging
import re
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import or_, select, text
@ -19,6 +19,11 @@ from app.db.models import (
ScholarProfile,
ScholarPublication,
)
from app.logging_utils import structured_log
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.doi.normalize import first_doi_from_texts
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion import safety as run_safety_service
from app.services.domains.ingestion.constants import (
FAILED_STATES,
FAILURE_BUCKET_BLOCKED,
@ -30,9 +35,6 @@ from app.services.domains.ingestion.constants import (
RESUMABLE_PARTIAL_REASONS,
RUN_LOCK_NAMESPACE,
)
from app.services.domains.arxiv.errors import ArxivRateLimitError
from app.services.domains.doi.normalize import first_doi_from_texts
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.ingestion.fingerprints import (
_build_body_excerpt,
_dedupe_publication_candidates,
@ -43,8 +45,6 @@ from app.services.domains.ingestion.fingerprints import (
canonical_title_for_dedup,
normalize_title,
)
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion import safety as run_safety_service
from app.services.domains.ingestion.types import (
PagedLoopState,
PagedParseResult,
@ -56,17 +56,17 @@ from app.services.domains.ingestion.types import (
RunProgress,
ScholarProcessingOutcome,
)
from app.services.domains.settings import application as user_settings_service
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.runs.events import run_events
from app.services.domains.scholar.parser import (
ParseState,
ParsedProfilePage,
ParseState,
PublicationCandidate,
ScholarParserError,
parse_profile_page,
)
from app.services.domains.scholar.source import FetchResult, ScholarSource
from app.logging_utils import structured_log
from app.services.domains.settings import application as user_settings_service
from app.settings import settings
logger = logging.getLogger(__name__)
@ -147,18 +147,20 @@ class ScholarIngestionService:
user_id: int,
trigger_type: RunTriggerType,
) -> None:
now_utc = datetime.now(timezone.utc)
now_utc = datetime.now(UTC)
previous = run_safety_service.get_safety_event_context(user_settings, now_utc=now_utc)
if run_safety_service.clear_expired_cooldown(user_settings, now_utc=now_utc):
await db_session.commit()
await db_session.refresh(user_settings)
structured_log(
logger, "info", "ingestion.cooldown_cleared",
logger,
"info",
"ingestion.cooldown_cleared",
user_id=user_id,
reason=previous.get("cooldown_reason"),
cooldown_until=previous.get("cooldown_until"),
)
now_utc = datetime.now(timezone.utc)
now_utc = datetime.now(UTC)
if run_safety_service.is_cooldown_active(user_settings, now_utc=now_utc):
await self._raise_safety_blocked_start(
db_session,
@ -183,7 +185,9 @@ class ScholarIngestionService:
)
await db_session.commit()
structured_log(
logger, "warning", "ingestion.safety_policy_blocked_run_start",
logger,
"warning",
"ingestion.safety_policy_blocked_run_start",
user_id=user_id,
trigger_type=trigger_type.value,
reason=safety_state.get("cooldown_reason"),
@ -204,14 +208,9 @@ class ScholarIngestionService:
start_cstart_by_scholar_id: dict[int, int] | None,
) -> tuple[set[int] | None, dict[int, int]]:
filtered_scholar_ids = (
{int(value) for value in scholar_profile_ids}
if scholar_profile_ids is not None
else None
{int(value) for value in scholar_profile_ids} if scholar_profile_ids is not None else None
)
start_cstart_map = {
int(key): max(0, int(value))
for key, value in (start_cstart_by_scholar_id or {}).items()
}
start_cstart_map = {int(key): max(0, int(value)) for key, value in (start_cstart_by_scholar_id or {}).items()}
return filtered_scholar_ids, start_cstart_map
async def _load_target_scholars(
@ -521,7 +520,9 @@ class ScholarIngestionService:
page_logs=paged_parse_result.page_logs,
)
structured_log(
logger, "warning", "ingestion.scholar_parse_failed",
logger,
"warning",
"ingestion.scholar_parse_failed",
scholar_profile_id=scholar.id,
scholar_id=scholar.scholar_id,
state=paged_parse_result.parsed_page.state.value,
@ -679,7 +680,7 @@ class ScholarIngestionService:
max_pages_per_scholar: int,
page_size: int,
) -> tuple[datetime, PagedParseResult, dict[str, Any]]:
run_dt = datetime.now(timezone.utc)
run_dt = datetime.now(UTC)
paged_parse_result = await self._fetch_and_parse_all_pages_with_retry(
scholar=scholar,
run=run,
@ -698,7 +699,9 @@ class ScholarIngestionService:
self._apply_first_page_profile_metadata(scholar=scholar, paged_parse_result=paged_parse_result, run_dt=run_dt)
parsed_page = paged_parse_result.parsed_page
structured_log(
logger, "info", "ingestion.scholar_parsed",
logger,
"info",
"ingestion.scholar_parsed",
user_id=user_id,
crawl_run_id=run.id,
scholar_profile_id=scholar.id,
@ -796,9 +799,7 @@ class ScholarIngestionService:
scholar_results=progress.scholar_results,
scholar_profile_id=scholar_profile_id,
)
next_succeeded, next_failed, next_partial = ScholarIngestionService._result_counters(
outcome.result_entry
)
next_succeeded, next_failed, next_partial = ScholarIngestionService._result_counters(outcome.result_entry)
if prior_index is None:
progress.scholar_results.append(outcome.result_entry)
ScholarIngestionService._adjust_progress_counts(
@ -809,9 +810,7 @@ class ScholarIngestionService:
)
return
previous_entry = progress.scholar_results[prior_index]
prev_succeeded, prev_failed, prev_partial = ScholarIngestionService._result_counters(
previous_entry
)
prev_succeeded, prev_failed, prev_partial = ScholarIngestionService._result_counters(previous_entry)
progress.scholar_results[prior_index] = outcome.result_entry
ScholarIngestionService._adjust_progress_counts(
progress=progress,
@ -829,7 +828,7 @@ class ScholarIngestionService:
exc: Exception,
) -> ScholarProcessingOutcome:
scholar.last_run_status = RunStatus.FAILED
scholar.last_run_dt = datetime.now(timezone.utc)
scholar.last_run_dt = datetime.now(UTC)
logger.exception(
"ingestion.scholar_unexpected_failure",
extra={
@ -933,7 +932,7 @@ class ScholarIngestionService:
) -> None:
pre_apply_state = run_safety_service.get_safety_event_context(
user_settings,
now_utc=datetime.now(timezone.utc),
now_utc=datetime.now(UTC),
)
safety_state, cooldown_trigger_reason = run_safety_service.apply_run_safety_outcome(
user_settings,
@ -944,11 +943,13 @@ class ScholarIngestionService:
network_failure_threshold=alert_summary.network_failure_threshold,
blocked_cooldown_seconds=settings.ingestion_safety_cooldown_blocked_seconds,
network_cooldown_seconds=settings.ingestion_safety_cooldown_network_seconds,
now_utc=datetime.now(timezone.utc),
now_utc=datetime.now(UTC),
)
if cooldown_trigger_reason is not None:
structured_log(
logger, "warning", "ingestion.safety_cooldown_entered",
logger,
"warning",
"ingestion.safety_cooldown_entered",
user_id=user_id,
crawl_run_id=int(run.id),
reason=cooldown_trigger_reason,
@ -962,7 +963,9 @@ class ScholarIngestionService:
)
elif pre_apply_state.get("cooldown_active") and not safety_state.get("cooldown_active"):
structured_log(
logger, "info", "ingestion.cooldown_cleared",
logger,
"info",
"ingestion.cooldown_cleared",
user_id=user_id,
crawl_run_id=int(run.id),
reason=pre_apply_state.get("cooldown_reason"),
@ -979,7 +982,7 @@ class ScholarIngestionService:
alert_summary: RunAlertSummary,
idempotency_key: str | None,
) -> None:
run.end_dt = datetime.now(timezone.utc)
run.end_dt = datetime.now(UTC)
if run.status != RunStatus.CANCELED:
run.status = self._resolve_run_status(
scholar_count=len(scholars),
@ -1062,7 +1065,26 @@ class ScholarIngestionService:
new_publication_count=run.new_pub_count,
)
async def _initialize_run_for_user(self, db_session: AsyncSession, *, user_id: int, trigger_type: RunTriggerType, scholar_profile_ids: set[int] | None, start_cstart_by_scholar_id: dict[int, int] | None, request_delay_seconds: int, network_error_retries: int, retry_backoff_seconds: float, rate_limit_retries: int, rate_limit_backoff_seconds: float, max_pages_per_scholar: int, page_size: int, idempotency_key: str | None, alert_blocked_failure_threshold: int, alert_network_failure_threshold: int, alert_retry_scheduled_threshold: int) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]:
async def _initialize_run_for_user(
self,
db_session: AsyncSession,
*,
user_id: int,
trigger_type: RunTriggerType,
scholar_profile_ids: set[int] | None,
start_cstart_by_scholar_id: dict[int, int] | None,
request_delay_seconds: int,
network_error_retries: int,
retry_backoff_seconds: float,
rate_limit_retries: int,
rate_limit_backoff_seconds: float,
max_pages_per_scholar: int,
page_size: int,
idempotency_key: str | None,
alert_blocked_failure_threshold: int,
alert_network_failure_threshold: int,
alert_retry_scheduled_threshold: int,
) -> tuple[Any, CrawlRun, list[ScholarProfile], dict[int, int]]:
user_settings = await self._load_user_settings_for_run(
db_session,
user_id=user_id,
@ -1116,7 +1138,9 @@ class ScholarIngestionService:
alert_retry_scheduled_threshold: int,
) -> CrawlRun:
structured_log(
logger, "info", "ingestion.run_started",
logger,
"info",
"ingestion.run_started",
user_id=user_id,
trigger_type=trigger_type.value,
scholar_count=len(scholars),
@ -1143,9 +1167,7 @@ class ScholarIngestionService:
except IntegrityError as exc:
if _is_active_run_integrity_error(exc):
await db_session.rollback()
raise RunAlreadyInProgressError(
f"Run already in progress for user_id={user_id}."
) from exc
raise RunAlreadyInProgressError(f"Run already in progress for user_id={user_id}.") from exc
raise
return run
@ -1177,8 +1199,11 @@ class ScholarIngestionService:
await db_session.refresh(run)
if run.status == RunStatus.CANCELED:
structured_log(
logger, "info", "ingestion.run_canceled",
run_id=run.id, user_id=user_id,
logger,
"info",
"ingestion.run_canceled",
run_id=run.id,
user_id=user_id,
)
return progress
await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds)
@ -1217,8 +1242,11 @@ class ScholarIngestionService:
await db_session.refresh(run)
if run.status == RunStatus.CANCELED:
structured_log(
logger, "info", "ingestion.run_canceled",
run_id=run.id, user_id=user_id,
logger,
"info",
"ingestion.run_canceled",
run_id=run.id,
user_id=user_id,
)
break
await self._wait_between_scholars(index=index, request_delay_seconds=request_delay_seconds)
@ -1263,7 +1291,9 @@ class ScholarIngestionService:
)
if alert_summary.alert_flags["blocked_failure_threshold_exceeded"]:
structured_log(
logger, "warning", "ingestion.alert_blocked_failure_threshold_exceeded",
logger,
"warning",
"ingestion.alert_blocked_failure_threshold_exceeded",
user_id=user_id,
crawl_run_id=int(run.id),
blocked_failure_count=alert_summary.blocked_failure_count,
@ -1271,7 +1301,9 @@ class ScholarIngestionService:
)
if alert_summary.alert_flags["network_failure_threshold_exceeded"]:
structured_log(
logger, "warning", "ingestion.alert_network_failure_threshold_exceeded",
logger,
"warning",
"ingestion.alert_network_failure_threshold_exceeded",
user_id=user_id,
crawl_run_id=int(run.id),
network_failure_count=alert_summary.network_failure_count,
@ -1279,7 +1311,9 @@ class ScholarIngestionService:
)
if alert_summary.alert_flags["retry_scheduled_threshold_exceeded"]:
structured_log(
logger, "warning", "ingestion.alert_retry_scheduled_threshold_exceeded",
logger,
"warning",
"ingestion.alert_retry_scheduled_threshold_exceeded",
user_id=user_id,
crawl_run_id=int(run.id),
retries_scheduled_count=failure_summary.retries_scheduled_count,
@ -1319,7 +1353,9 @@ class ScholarIngestionService:
effective_delay = self._effective_request_delay_seconds(request_delay_seconds)
if effective_delay != _int_or_default(request_delay_seconds, effective_delay):
structured_log(
logger, "warning", "ingestion.delay_coerced",
logger,
"warning",
"ingestion.delay_coerced",
user_id=user_id,
requested_request_delay_seconds=_int_or_default(request_delay_seconds, 0),
effective_request_delay_seconds=effective_delay,
@ -1332,8 +1368,12 @@ class ScholarIngestionService:
request_delay_seconds=effective_delay,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries if rate_limit_retries is not None else settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds if rate_limit_backoff_seconds is not None else settings.ingestion_rate_limit_backoff_seconds,
rate_limit_retries=rate_limit_retries
if rate_limit_retries is not None
else settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds
if rate_limit_backoff_seconds is not None
else settings.ingestion_rate_limit_backoff_seconds,
max_pages_per_scholar=max_pages_per_scholar,
page_size=page_size,
)
@ -1383,13 +1423,17 @@ class ScholarIngestionService:
run, user_settings, attached_scholars = await self._prepare_execute_run(
db_session, run_id=run_id, user_id=user_id, scholars=scholars
)
paging_kwargs = self._paging_kwargs(
request_delay_seconds=request_delay_seconds,
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries if rate_limit_retries is not None else settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds if rate_limit_backoff_seconds is not None else settings.ingestion_rate_limit_backoff_seconds,
rate_limit_retries=rate_limit_retries
if rate_limit_retries is not None
else settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds
if rate_limit_backoff_seconds is not None
else settings.ingestion_rate_limit_backoff_seconds,
max_pages_per_scholar=max_pages_per_scholar,
page_size=page_size,
)
@ -1427,7 +1471,9 @@ class ScholarIngestionService:
run.status = RunStatus.RESOLVING
await db_session.commit()
structured_log(
logger, "info", "ingestion.run_completed",
logger,
"info",
"ingestion.run_completed",
user_id=user_id,
crawl_run_id=run.id,
status=run.status.value,
@ -1471,7 +1517,8 @@ class ScholarIngestionService:
scholar_ids = [s.id for s in scholars]
scholars_result = await db_session.execute(
select(ScholarProfile).where(ScholarProfile.id.in_(scholar_ids))
select(ScholarProfile)
.where(ScholarProfile.id.in_(scholar_ids))
.order_by(ScholarProfile.created_at.asc(), ScholarProfile.id.asc())
)
return run, user_settings, list(scholars_result.scalars().all())
@ -1481,7 +1528,7 @@ class ScholarIngestionService:
run_to_fail = await cleanup_session.get(CrawlRun, run_id)
if run_to_fail:
run_to_fail.status = RunStatus.FAILED
run_to_fail.end_dt = datetime.now(timezone.utc)
run_to_fail.end_dt = datetime.now(UTC)
run_to_fail.error_log["terminal_exception"] = str(exc)
await cleanup_session.commit()
@ -1568,17 +1615,21 @@ class ScholarIngestionService:
alert_network_failure_threshold=alert_network_failure_threshold,
alert_retry_scheduled_threshold=alert_retry_scheduled_threshold,
)
paging_kwargs = self._paging_kwargs(
request_delay_seconds=self._effective_request_delay_seconds(request_delay_seconds),
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries if rate_limit_retries is not None else settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds if rate_limit_backoff_seconds is not None else settings.ingestion_rate_limit_backoff_seconds,
rate_limit_retries=rate_limit_retries
if rate_limit_retries is not None
else settings.ingestion_rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds
if rate_limit_backoff_seconds is not None
else settings.ingestion_rate_limit_backoff_seconds,
max_pages_per_scholar=max_pages_per_scholar,
page_size=page_size,
)
threshold_kwargs = self._threshold_kwargs(
alert_blocked_failure_threshold=alert_blocked_failure_threshold,
alert_network_failure_threshold=alert_network_failure_threshold,
@ -1628,7 +1679,9 @@ class ScholarIngestionService:
await db_session.commit()
structured_log(
logger, "info", "ingestion.run_completed",
logger,
"info",
"ingestion.run_completed",
user_id=user_id,
crawl_run_id=run.id,
status=run.status.value,
@ -1663,8 +1716,7 @@ class ScholarIngestionService:
return await self._source.fetch_profile_html(scholar_id)
return FetchResult(
requested_url=(
"https://scholar.google.com/citations"
f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
f"https://scholar.google.com/citations?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
),
status_code=None,
final_url=None,
@ -1682,8 +1734,7 @@ class ScholarIngestionService:
)
return FetchResult(
requested_url=(
"https://scholar.google.com/citations"
f"?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
f"https://scholar.google.com/citations?hl=en&user={scholar_id}&cstart={cstart}&pagesize={page_size}"
),
status_code=None,
final_url=None,
@ -1702,7 +1753,10 @@ class ScholarIngestionService:
) -> bool:
if parsed_page.state == ParseState.NETWORK_ERROR:
return network_attempt_count <= network_error_retries
if parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited":
if (
parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
and parsed_page.state_reason == "blocked_http_429_rate_limited"
):
return rate_limit_attempt_count <= rate_limit_retries
return False
@ -1725,7 +1779,9 @@ class ScholarIngestionService:
attempt_label = network_attempt_count
structured_log(
logger, "warning", "ingestion.scholar_retry_scheduled",
logger,
"warning",
"ingestion.scholar_retry_scheduled",
scholar_id=scholar_id,
cstart=cstart,
attempt_count=attempt_label,
@ -1759,24 +1815,29 @@ class ScholarIngestionService:
page_size=page_size,
)
parsed_page = self._parse_profile_page_or_layout_error(fetch_result=fetch_result)
if parsed_page.state == ParseState.NETWORK_ERROR:
network_attempts += 1
total_attempts = network_attempts
elif parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA and parsed_page.state_reason == "blocked_http_429_rate_limited":
elif (
parsed_page.state == ParseState.BLOCKED_OR_CAPTCHA
and parsed_page.state_reason == "blocked_http_429_rate_limited"
):
rate_limit_attempts += 1
total_attempts = rate_limit_attempts
else:
total_attempts = network_attempts + rate_limit_attempts + 1
attempt_log.append({
"attempt": total_attempts,
"cstart": cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"fetch_error": fetch_result.error,
})
attempt_log.append(
{
"attempt": total_attempts,
"cstart": cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"fetch_error": fetch_result.error,
}
)
if not self._should_retry_page_fetch(
parsed_page=parsed_page,
@ -1995,18 +2056,20 @@ class ScholarIngestionService:
) -> None:
state.pages_attempted += 1
state.attempt_log.extend(page_attempt_log)
state.page_logs.append({
"page": state.pages_attempted,
"cstart": state.current_cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"publication_count": len(parsed_page.publications),
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"warning_codes": parsed_page.warnings,
"attempt_count": len(page_attempt_log),
})
state.page_logs.append(
{
"page": state.pages_attempted,
"cstart": state.current_cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"publication_count": len(parsed_page.publications),
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"warning_codes": parsed_page.warnings,
"attempt_count": len(page_attempt_log),
}
)
state.fetch_result = fetch_result
state.parsed_page = parsed_page
@ -2052,18 +2115,20 @@ class ScholarIngestionService:
)
first_page_fingerprint_sha256 = build_initial_page_fingerprint(parsed_page)
attempt_log = list(first_attempt_log)
page_logs = [{
"page": 1,
"cstart": start_cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"publication_count": len(parsed_page.publications),
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"warning_codes": parsed_page.warnings,
"attempt_count": len(first_attempt_log),
}]
page_logs = [
{
"page": 1,
"cstart": start_cstart,
"state": parsed_page.state.value,
"state_reason": parsed_page.state_reason,
"status_code": fetch_result.status_code,
"publication_count": len(parsed_page.publications),
"articles_range": parsed_page.articles_range,
"has_show_more_button": parsed_page.has_show_more_button,
"warning_codes": parsed_page.warnings,
"attempt_count": len(first_attempt_log),
}
]
return fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs
async def _paginate_loop(
@ -2098,7 +2163,9 @@ class ScholarIngestionService:
await db_session.refresh(run)
if run.status == RunStatus.CANCELED:
structured_log(
logger, "info", "ingestion.pagination_canceled",
logger,
"info",
"ingestion.pagination_canceled",
run_id=run.id,
)
self._set_truncated_state(
@ -2217,12 +2284,17 @@ class ScholarIngestionService:
rate_limit_backoff_seconds: float,
max_pages: int,
page_size: int,
previous_initial_page_fingerprint_sha256: str | None = None
previous_initial_page_fingerprint_sha256: str | None = None,
) -> PagedParseResult:
bounded_max_pages = max(1, int(max_pages))
bounded_page_size = max(1, int(page_size))
fetch_result, parsed_page, first_page_fingerprint_sha256, attempt_log, page_logs = (
await self._fetch_initial_page_context(
(
fetch_result,
parsed_page,
first_page_fingerprint_sha256,
attempt_log,
page_logs,
) = await self._fetch_initial_page_context(
scholar_id=scholar.scholar_id,
start_cstart=start_cstart,
bounded_page_size=bounded_page_size,
@ -2230,7 +2302,7 @@ class ScholarIngestionService:
retry_backoff_seconds=retry_backoff_seconds,
rate_limit_retries=rate_limit_retries,
rate_limit_backoff_seconds=rate_limit_backoff_seconds,
))
)
shortcut_result = self._short_circuit_initial_page(
start_cstart=start_cstart,
previous_initial_page_fingerprint_sha256=previous_initial_page_fingerprint_sha256,
@ -2275,9 +2347,7 @@ class ScholarIngestionService:
*,
run_id: int,
) -> bool:
check_result = await db_session.execute(
select(CrawlRun.status).where(CrawlRun.id == run_id)
)
check_result = await db_session.execute(select(CrawlRun.status).where(CrawlRun.id == run_id))
status = check_result.scalar_one_or_none()
if status is None:
raise RuntimeError(f"Missing crawl_run for run_id={run_id}.")
@ -2302,12 +2372,10 @@ class ScholarIngestionService:
)
from app.services.domains.openalex.matching import find_best_match
run_result = await db_session.execute(
select(CrawlRun.user_id).where(CrawlRun.id == run_id)
)
run_result = await db_session.execute(select(CrawlRun.user_id).where(CrawlRun.id == run_id))
user_id = run_result.scalar_one()
now = datetime.now(timezone.utc)
now = datetime.now(UTC)
cooldown_threshold = now - timedelta(days=7)
stmt = (
@ -2355,21 +2423,28 @@ class ScholarIngestionService:
)
except OpenAlexBudgetExhaustedError:
structured_log(
logger, "warning", "ingestion.openalex_budget_exhausted",
logger,
"warning",
"ingestion.openalex_budget_exhausted",
run_id=run_id,
)
break # Stop all enrichment — budget won't reset until midnight UTC
except OpenAlexRateLimitError:
structured_log(
logger, "warning", "ingestion.openalex_rate_limited",
logger,
"warning",
"ingestion.openalex_rate_limited",
run_id=run_id,
)
await asyncio.sleep(60)
continue
except Exception as e:
structured_log(
logger, "warning", "ingestion.openalex_enrichment_failed",
error=str(e), run_id=run_id,
logger,
"warning",
"ingestion.openalex_enrichment_failed",
error=str(e),
run_id=run_id,
)
continue
@ -2406,7 +2481,9 @@ class ScholarIngestionService:
merge_count = await sweep_identifier_duplicates(db_session)
if merge_count:
structured_log(
logger, "info", "ingestion.identifier_dedup_sweep",
logger,
"info",
"ingestion.identifier_dedup_sweep",
merged_count=merge_count,
run_id=run_id,
)
@ -2444,7 +2521,9 @@ class ScholarIngestionService:
return True
except ArxivRateLimitError:
structured_log(
logger, "warning", "ingestion.arxiv_rate_limited",
logger,
"warning",
"ingestion.arxiv_rate_limited",
run_id=run_id,
publication_id=int(publication.id),
detail="arXiv temporarily disabled for remaining enrichment pass",
@ -2498,17 +2577,18 @@ class ScholarIngestionService:
from app.services.domains.openalex.client import OpenAlexClient
from app.services.domains.openalex.matching import find_best_match
client = OpenAlexClient(api_key=settings.openalex_api_key, mailto=settings.crossref_api_mailto)
# Batch requests by 25 to stay within URL length limits
batch_size = 25
enriched: list[PublicationCandidate] = []
import re
for i in range(0, len(publications), batch_size):
batch = publications[i:i + batch_size]
batch = publications[i : i + batch_size]
titles = []
for p in batch:
if not p.title:
@ -2519,29 +2599,27 @@ class ScholarIngestionService:
safe_title = " ".join(safe_title.split())
if safe_title:
titles.append(safe_title)
if not titles:
enriched.extend(batch)
continue
query = "|".join(t for t in titles)
try:
openalex_works = await client.get_works_by_filter(
{"title.search": query}, limit=batch_size * 3
)
openalex_works = await client.get_works_by_filter({"title.search": query}, limit=batch_size * 3)
except Exception as e:
logger.warning(
"ingestion.openalex_enrichment_failed",
extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id}
extra={"error": str(e), "batch_size": len(batch), "scholar_id": scholar.id},
)
openalex_works = []
for p in batch:
match = find_best_match(
target_title=p.title,
target_year=p.year,
target_authors=p.authors_text or (scholar.display_name or scholar.scholar_id),
candidates=openalex_works
candidates=openalex_works,
)
if match:
new_p = PublicationCandidate(
@ -2627,7 +2705,7 @@ class ScholarIngestionService:
"pub_url": publication.pub_url,
"scholar_profile_id": scholar.id,
"scholar_label": scholar.display_name or scholar.scholar_id,
"first_seen_at": datetime.now(timezone.utc).isoformat(),
"first_seen_at": datetime.now(UTC).isoformat(),
"new_publication_count": int(run.new_pub_count or 0),
},
)
@ -2647,9 +2725,7 @@ class ScholarIngestionService:
) -> Publication | None:
if not cluster_id:
return None
result = await db_session.execute(
select(Publication).where(Publication.cluster_id == cluster_id)
)
result = await db_session.execute(select(Publication).where(Publication.cluster_id == cluster_id))
return result.scalar_one_or_none()
async def _find_publication_by_fingerprint(
@ -2658,9 +2734,7 @@ class ScholarIngestionService:
*,
fingerprint: str,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(Publication.fingerprint_sha256 == fingerprint)
)
result = await db_session.execute(select(Publication).where(Publication.fingerprint_sha256 == fingerprint))
return result.scalar_one_or_none()
@staticmethod
@ -2685,9 +2759,7 @@ class ScholarIngestionService:
canonical_title_hash: str,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(
Publication.canonical_title_hash == canonical_title_hash
)
select(Publication).where(Publication.canonical_title_hash == canonical_title_hash)
)
return result.scalar_one_or_none()
@ -2816,9 +2888,7 @@ class ScholarIngestionService:
) -> tuple[str | None, int]:
if outcome == "partial":
reason = (pagination_truncated_reason or "").strip()
if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(
RESUMABLE_PARTIAL_REASON_PREFIXES
):
if reason in RESUMABLE_PARTIAL_REASONS or reason.startswith(RESUMABLE_PARTIAL_REASON_PREFIXES):
return reason, queue_service.normalize_cstart(
continuation_cstart if continuation_cstart is not None else fallback_cstart
)
@ -2852,13 +2922,9 @@ class ScholarIngestionService:
"has_show_more_button": parsed_page.has_show_more_button,
"has_operation_error_banner": parsed_page.has_operation_error_banner,
"warning_codes": parsed_page.warnings,
"marker_counts_nonzero": {
key: value for key, value in parsed_page.marker_counts.items() if value > 0
},
"marker_counts_nonzero": {key: value for key, value in parsed_page.marker_counts.items() if value > 0},
"body_length": len(fetch_result.body),
"body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest()
if fetch_result.body
else None,
"body_sha256": hashlib.sha256(fetch_result.body.encode("utf-8")).hexdigest() if fetch_result.body else None,
"body_excerpt": _build_body_excerpt(fetch_result.body),
"attempt_log": attempt_log,
}
@ -2876,9 +2942,7 @@ class ScholarIngestionService:
user_id: int,
) -> bool:
result = await db_session.execute(
text(
"SELECT pg_try_advisory_xact_lock(:namespace, :user_key)"
),
text("SELECT pg_try_advisory_xact_lock(:namespace, :user_key)"),
{
"namespace": RUN_LOCK_NAMESPACE,
"user_key": int(user_id),

6
app/services/domains/ingestion/fingerprints.py
View file

@ -3,8 +3,8 @@ from __future__ import annotations
import hashlib
import json
import re
from typing import Any
import unicodedata
from typing import Any
from urllib.parse import urljoin
from app.services.domains.ingestion.constants import (
@ -14,7 +14,7 @@ from app.services.domains.ingestion.constants import (
TITLE_ALNUM_RE,
WORD_RE,
)
from app.services.domains.scholar.parser import ParseState, ParsedProfilePage, PublicationCandidate
from app.services.domains.scholar.parser import ParsedProfilePage, ParseState, PublicationCandidate
# Scholar-specific noise patterns stripped before canonical comparison.
# Applied in order; each targets a different Scholar metadata injection style.
@ -381,4 +381,4 @@ def _build_body_excerpt(body: str, *, max_chars: int = 220) -> str | None:
return None
if len(flattened) <= max_chars:
return flattened
return f"{flattened[:max_chars - 1]}..."
return f"{flattened[: max_chars - 1]}..."

44
app/services/domains/ingestion/queue.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@ -115,7 +115,7 @@ async def upsert_job(
run_id: int | None,
delay_seconds: int,
) -> IngestionQueueItem:
now = datetime.now(timezone.utc)
now = datetime.now(UTC)
next_attempt_dt = now + timedelta(seconds=max(0, int(delay_seconds)))
item = await _get_item_for_user_scholar(
db_session,
@ -171,9 +171,7 @@ async def delete_job_by_id(
*,
job_id: int,
) -> bool:
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return False
@ -222,10 +220,8 @@ async def increment_attempt_count(
*,
job_id: int,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
@ -239,10 +235,8 @@ async def reset_attempt_count(
*,
job_id: int,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
@ -259,10 +253,8 @@ async def reschedule_job(
reason: str,
error: str | None = None,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
@ -282,10 +274,8 @@ async def mark_retrying(
job_id: int,
reason: str | None = None,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
@ -305,10 +295,8 @@ async def mark_dropped(
reason: str,
error: str | None = None,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None
@ -329,10 +317,8 @@ async def mark_queued_now(
reason: str,
reset_attempt_count: bool = False,
) -> IngestionQueueItem | None:
now = datetime.now(timezone.utc)
result = await db_session.execute(
select(IngestionQueueItem).where(IngestionQueueItem.id == job_id)
)
now = datetime.now(UTC)
result = await db_session.execute(select(IngestionQueueItem).where(IngestionQueueItem.id == job_id))
item = result.scalar_one_or_none()
if item is None:
return None

20
app/services/domains/ingestion/safety.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta
from typing import Any
from app.db.models import UserSetting
@ -18,7 +18,7 @@ _COUNTER_LAST_EVALUATED_RUN_ID = "last_evaluated_run_id"
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
return datetime.now(UTC)
def _safe_int(value: Any, default: int = 0) -> int:
@ -41,8 +41,8 @@ def _normalize_datetime(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
def _state_dict(settings: UserSetting) -> dict[str, Any]:
@ -100,9 +100,7 @@ def _recommended_action(reason: str | None) -> str | None:
"increase request delay, and avoid repeated manual retries."
)
if reason == COOLDOWN_REASON_NETWORK_FAILURE_THRESHOLD:
return (
"Network failures crossed the threshold. Verify connectivity and retry after cooldown."
)
return "Network failures crossed the threshold. Verify connectivity and retry after cooldown."
return None
@ -159,14 +157,10 @@ def _update_run_counters(
counters[_COUNTER_LAST_NETWORK_FAILURE_COUNT] = bounded_network_failures
counters[_COUNTER_LAST_EVALUATED_RUN_ID] = int(run_id)
counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1
if bounded_blocked_failures > 0
else 0
int(counters[_COUNTER_CONSECUTIVE_BLOCKED_RUNS]) + 1 if bounded_blocked_failures > 0 else 0
)
counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS] = (
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1
if bounded_network_failures > 0
else 0
int(counters[_COUNTER_CONSECUTIVE_NETWORK_RUNS]) + 1 if bounded_network_failures > 0 else 0
)
return bounded_blocked_failures, bounded_network_failures

139
app/services/domains/ingestion/scheduler.py
View file

@ -1,9 +1,9 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
import logging
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@ -17,24 +17,22 @@ from app.db.models import (
UserSetting,
)
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.domains.ingestion import queue as queue_service
from app.services.domains.ingestion.application import (
RunAlreadyInProgressError,
RunBlockedBySafetyPolicyError,
ScholarIngestionService,
)
from app.logging_utils import structured_log
from app.services.domains.settings import application as user_settings_service
from app.services.domains.scholar.source import LiveScholarSource
from app.services.domains.settings import application as user_settings_service
from app.settings import settings
logger = logging.getLogger(__name__)
def _request_delay_floor_seconds() -> int:
return user_settings_service.resolve_request_delay_minimum(
settings.ingestion_min_request_delay_seconds
)
return user_settings_service.resolve_request_delay_minimum(settings.ingestion_min_request_delay_seconds)
def _effective_request_delay_seconds(value: int | None) -> int:
@ -96,7 +94,9 @@ class SchedulerService:
return
self._task = asyncio.create_task(self._run_loop(), name="scholarr-scheduler")
structured_log(
logger, "info", "scheduler.started",
logger,
"info",
"scheduler.started",
tick_seconds=self._tick_seconds,
network_error_retries=self._network_error_retries,
retry_backoff_seconds=self._retry_backoff_seconds,
@ -137,13 +137,13 @@ class SchedulerService:
async def _tick_once(self) -> None:
if self._continuation_queue_enabled:
await self._drain_continuation_queue()
await self._drain_pdf_queue()
candidates = await self._load_candidates()
if not candidates:
return
now = datetime.now(timezone.utc)
now = datetime.now(UTC)
for candidate in candidates:
if not await self._is_due(candidate, now=now):
continue
@ -170,10 +170,12 @@ class SchedulerService:
def _candidate_from_row(row: tuple, *, now_utc: datetime) -> _AutoRunCandidate | None:
user_id, run_interval_minutes, request_delay_seconds, cooldown_until, cooldown_reason = row
if cooldown_until is not None and cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
cooldown_until = cooldown_until.replace(tzinfo=UTC)
if cooldown_until is not None and cooldown_until > now_utc:
structured_log(
logger, "info", "scheduler.run_skipped_safety_cooldown_precheck",
logger,
"info",
"scheduler.run_skipped_safety_cooldown_precheck",
user_id=int(user_id),
reason=cooldown_reason,
cooldown_until=cooldown_until,
@ -192,7 +194,7 @@ class SchedulerService:
if not settings.ingestion_automation_allowed:
return []
rows = await self._load_candidate_rows()
now_utc = datetime.now(timezone.utc)
now_utc = datetime.now(UTC)
candidates: list[_AutoRunCandidate] = []
for row in rows:
candidate = self._candidate_from_row(row, now_utc=now_utc)
@ -216,9 +218,7 @@ class SchedulerService:
if last_run is None:
return True
next_due_dt = last_run + timedelta(
minutes=candidate.run_interval_minutes
)
next_due_dt = last_run + timedelta(minutes=candidate.run_interval_minutes)
return now >= next_due_dt
async def _run_candidate_ingestion(
@ -252,7 +252,9 @@ class SchedulerService:
except RunBlockedBySafetyPolicyError as exc:
await session.rollback()
structured_log(
logger, "info", "scheduler.run_skipped_safety_cooldown",
logger,
"info",
"scheduler.run_skipped_safety_cooldown",
user_id=candidate.user_id,
reason=exc.safety_state.get("cooldown_reason"),
cooldown_until=exc.safety_state.get("cooldown_until"),
@ -269,7 +271,9 @@ class SchedulerService:
if run_summary is None:
return
structured_log(
logger, "info", "scheduler.run_completed",
logger,
"info",
"scheduler.run_completed",
user_id=candidate.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
@ -278,7 +282,7 @@ class SchedulerService:
)
async def _drain_continuation_queue(self) -> None:
now = datetime.now(timezone.utc)
now = datetime.now(UTC)
session_factory = get_session_factory()
async with session_factory() as session:
jobs = await queue_service.list_due_jobs(
@ -305,7 +309,9 @@ class SchedulerService:
await session.commit()
if dropped is not None:
structured_log(
logger, "warning", "scheduler.queue_item_dropped_max_attempts",
logger,
"warning",
"scheduler.queue_item_dropped_max_attempts",
queue_item_id=job.id,
user_id=job.user_id,
scholar_profile_id=job.scholar_profile_id,
@ -352,7 +358,9 @@ class SchedulerService:
await session.commit()
if dropped is not None:
structured_log(
logger, "info", "scheduler.queue_item_dropped_scholar_unavailable",
logger,
"info",
"scheduler.queue_item_dropped_scholar_unavailable",
queue_item_id=job.id,
user_id=job.user_id,
scholar_profile_id=job.scholar_profile_id,
@ -371,8 +379,11 @@ class SchedulerService:
)
await recovery_session.commit()
structured_log(
logger, "info", "scheduler.queue_item_deferred_lock",
queue_item_id=job.id, user_id=job.user_id,
logger,
"info",
"scheduler.queue_item_deferred_lock",
queue_item_id=job.id,
user_id=job.user_id,
)
async def _reschedule_queue_job_safety_cooldown(
@ -395,7 +406,9 @@ class SchedulerService:
)
await recovery_session.commit()
structured_log(
logger, "info", "scheduler.queue_item_deferred_safety_cooldown",
logger,
"info",
"scheduler.queue_item_deferred_safety_cooldown",
queue_item_id=job.id,
user_id=job.user_id,
reason=exc.safety_state.get("cooldown_reason"),
@ -423,8 +436,12 @@ class SchedulerService:
)
await recovery_session.commit()
structured_log(
logger, "warning", "scheduler.queue_item_dropped_after_exception",
queue_item_id=job.id, user_id=job.user_id, attempt_count=queue_item.attempt_count,
logger,
"warning",
"scheduler.queue_item_dropped_after_exception",
queue_item_id=job.id,
user_id=job.user_id,
attempt_count=queue_item.attempt_count,
)
return
delay_seconds = queue_service.compute_backoff_seconds(
@ -489,15 +506,23 @@ class SchedulerService:
await session.commit()
if queue_item is None:
structured_log(
logger, "info", "scheduler.queue_item_resolved",
queue_item_id=job.id, user_id=job.user_id,
run_id=run_summary.crawl_run_id, status=run_summary.status.value,
logger,
"info",
"scheduler.queue_item_resolved",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
)
return
structured_log(
logger, "info", "scheduler.queue_item_progressed",
queue_item_id=job.id, user_id=job.user_id,
run_id=run_summary.crawl_run_id, status=run_summary.status.value,
logger,
"info",
"scheduler.queue_item_progressed",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
attempt_count=int(queue_item.attempt_count),
)
return
@ -505,28 +530,50 @@ class SchedulerService:
if queue_item is None:
await session.commit()
structured_log(
logger, "info", "scheduler.queue_item_resolved",
queue_item_id=job.id, user_id=job.user_id,
run_id=run_summary.crawl_run_id, status=run_summary.status.value,
logger,
"info",
"scheduler.queue_item_resolved",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
)
return
if int(queue_item.attempt_count) >= self._continuation_max_attempts:
await queue_service.mark_dropped(session, job_id=job.id, reason="max_attempts_after_run")
await session.commit()
structured_log(
logger, "warning", "scheduler.queue_item_dropped_max_attempts_after_run",
queue_item_id=job.id, user_id=job.user_id,
logger,
"warning",
"scheduler.queue_item_dropped_max_attempts_after_run",
queue_item_id=job.id,
user_id=job.user_id,
attempt_count=queue_item.attempt_count,
run_id=run_summary.crawl_run_id, status=run_summary.status.value,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
)
return
delay_seconds = queue_service.compute_backoff_seconds(base_seconds=self._continuation_base_delay_seconds, attempt_count=int(queue_item.attempt_count), max_seconds=self._continuation_max_delay_seconds)
await queue_service.reschedule_job(session, job_id=job.id, delay_seconds=delay_seconds, reason=queue_item.reason, error=queue_item.last_error)
delay_seconds = queue_service.compute_backoff_seconds(
base_seconds=self._continuation_base_delay_seconds,
attempt_count=int(queue_item.attempt_count),
max_seconds=self._continuation_max_delay_seconds,
)
await queue_service.reschedule_job(
session,
job_id=job.id,
delay_seconds=delay_seconds,
reason=queue_item.reason,
error=queue_item.last_error,
)
await session.commit()
structured_log(
logger, "info", "scheduler.queue_item_rescheduled_failed",
queue_item_id=job.id, user_id=job.user_id,
run_id=run_summary.crawl_run_id, status=run_summary.status.value,
logger,
"info",
"scheduler.queue_item_rescheduled_failed",
queue_item_id=job.id,
user_id=job.user_id,
run_id=run_summary.crawl_run_id,
status=run_summary.status.value,
attempt_count=int(queue_item.attempt_count),
delay_seconds=delay_seconds,
)
@ -545,7 +592,7 @@ class SchedulerService:
async def _drain_pdf_queue(self) -> None:
from app.services.domains.publications.pdf_queue import drain_ready_jobs
session_factory = get_session_factory()
async with session_factory() as session:
try:
@ -556,7 +603,9 @@ class SchedulerService:
)
if processed > 0:
structured_log(
logger, "info", "scheduler.pdf_queue_drain_completed",
logger,
"info",
"scheduler.pdf_queue_drain_completed",
processed_count=processed,
)
except Exception:

20
app/services/domains/openalex/client.py
View file

@ -1,6 +1,4 @@
import asyncio
import logging
from typing import Any, Mapping
import httpx
from tenacity import (
@ -23,11 +21,13 @@ class OpenAlexClientError(Exception):
class OpenAlexRateLimitError(OpenAlexClientError):
"""Transient rate limit (too many requests per second)."""
pass
class OpenAlexBudgetExhaustedError(OpenAlexClientError):
"""Daily API budget exhausted — retrying is futile until midnight UTC."""
pass
@ -64,7 +64,7 @@ class OpenAlexClient:
return None
url = f"{OPENALEX_BASE_URL}/works/{clean_doi}"
headers = {}
if self.mailto:
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
@ -79,9 +79,7 @@ class OpenAlexClient:
if response.status_code == 429:
remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
if remaining == "0" or remaining.startswith("-"):
raise OpenAlexBudgetExhaustedError(
"Daily API budget exhausted; retrying won't help until midnight UTC"
)
raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex work by DOI")
if response.status_code >= 400:
logger.warning("OpenAlex API error: %s %s", response.status_code, response.text[:500])
@ -110,13 +108,13 @@ class OpenAlexClient:
# Example: {"doi": "10.foo|10.bar", "title.search": "query"}
filter_str = ",".join(f"{k}:{v}" for k, v in filters.items())
params = self._base_params.copy()
params["filter"] = filter_str
params["per-page"] = str(limit)
url = f"{OPENALEX_BASE_URL}/works"
headers = {}
if self.mailto:
headers["User-Agent"] = f"scholar-scraper/1.0 (mailto:{self.mailto})"
@ -129,9 +127,7 @@ class OpenAlexClient:
if response.status_code == 429:
remaining = response.headers.get("X-RateLimit-Remaining-USD", "")
if remaining == "0" or remaining.startswith("-"):
raise OpenAlexBudgetExhaustedError(
"Daily API budget exhausted; retrying won't help until midnight UTC"
)
raise OpenAlexBudgetExhaustedError("Daily API budget exhausted; retrying won't help until midnight UTC")
raise OpenAlexRateLimitError("Rate limit exceeded fetching OpenAlex works list")
if response.status_code >= 400:
logger.warning("OpenAlex API error (filters=%s): %s %s", filters, response.status_code, response.text[:500])
@ -139,7 +135,7 @@ class OpenAlexClient:
data = response.json()
results = data.get("results") or []
parsed_works = []
for raw_work in results:
try:

27
app/services/domains/openalex/matching.py
View file

@ -24,11 +24,11 @@ def _clean_string(s: str | None) -> str:
def _author_overlap_score(target_authors: str | None, candidate_authors: list[str]) -> bool:
if not target_authors or not candidate_authors:
return False
target_clean = _clean_string(target_authors)
if not target_clean:
return False
for candidate in candidate_authors:
cand_clean = _clean_string(candidate)
if cand_clean and (cand_clean in target_clean or target_clean in cand_clean):
@ -36,7 +36,7 @@ def _author_overlap_score(target_authors: str | None, candidate_authors: list[st
# Alternatively check rapidfuzz token_set_ratio
if cand_clean and fuzz.token_set_ratio(target_clean, cand_clean) > 80:
return True
return False
@ -62,12 +62,12 @@ def find_best_match(
for cand in candidates:
if not cand.title:
continue
clean_cand = _clean_string(cand.title)
# Primary sort: string similarity ratio
score = fuzz.ratio(clean_target, clean_cand)
if score >= TITLE_MATCH_THRESHOLD:
scored_candidates.append((score, cand))
@ -76,13 +76,12 @@ def find_best_match(
# Sort descending by score
scored_candidates.sort(key=lambda x: x[0], reverse=True)
best_score = scored_candidates[0][0]
# Extract all candidates within the tiebreaker margin
top_scored_candidates = [
(score, cand) for score, cand in scored_candidates
if best_score - score <= TIEBREAKER_MARGIN
(score, cand) for score, cand in scored_candidates if best_score - score <= TIEBREAKER_MARGIN
]
if len(top_scored_candidates) == 1:
@ -91,18 +90,18 @@ def find_best_match(
# We have a tie or near-tie. Use year and author overlap to break the tie.
# Score candidates: +1 for year match (within 1 year), +1 for author overlap
tiebreaker_scores: list[tuple[int, float, OpenAlexWork]] = []
for original_score, cand in top_scored_candidates:
tb_score = 0
if target_year is not None and cand.publication_year is not None:
if abs(target_year - cand.publication_year) <= 1:
tb_score += 1
candidate_author_names = [a.display_name for a in cand.authors if a.display_name]
if _author_overlap_score(target_authors, candidate_author_names):
tb_score += 1
tiebreaker_scores.append((tb_score, original_score, cand))
tiebreaker_scores.sort(key=lambda x: (x[0], x[1]), reverse=True)
return tiebreaker_scores[0][2]

12
app/services/domains/openalex/types.py
View file

@ -1,8 +1,8 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Mapping
from typing import Any
@dataclass(frozen=True)
@ -28,24 +28,24 @@ class OpenAlexWork:
@classmethod
def from_api_dict(cls, data: Mapping[str, Any]) -> OpenAlexWork:
ids = data.get("ids") or {}
# Extract DOI without the https://doi.org/ prefix
doi = ids.get("doi")
if doi and doi.startswith("https://doi.org/"):
doi = doi[16:]
# Extract PMID without the url prefix
pmid = ids.get("pmid")
if pmid and pmid.startswith("https://pubmed.ncbi.nlm.nih.gov/"):
pmid = pmid[32:]
# Extract PMCID without the url prefix
pmcid = ids.get("pmcid")
if pmcid and pmcid.startswith("https://www.ncbi.nlm.nih.gov/pmc/articles/"):
pmcid = pmcid[42:]
open_access = data.get("open_access") or {}
authors = []
for authorship in data.get("authorships") or []:
author_data = authorship.get("author") or {}

4
app/services/domains/portability/application.py
View file

@ -18,7 +18,7 @@ from app.services.domains.portability.publication_import import (
_upsert_imported_publication,
)
from app.services.domains.portability.scholar_import import _upsert_imported_scholars
from app.services.domains.portability.types import ImportExportError, ImportedPublicationInput
from app.services.domains.portability.types import ImportedPublicationInput, ImportExportError
async def import_user_data(
@ -58,8 +58,8 @@ async def import_user_data(
__all__ = [
"EXPORT_SCHEMA_VERSION",
"MAX_IMPORT_SCHOLARS",
"MAX_IMPORT_PUBLICATIONS",
"MAX_IMPORT_SCHOLARS",
"ImportExportError",
"ImportedPublicationInput",
"export_user_data",

8
app/services/domains/portability/exporting.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timezone
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
@ -11,7 +11,7 @@ from app.services.domains.portability.constants import EXPORT_SCHEMA_VERSION
def _exported_at_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
return datetime.now(UTC).replace(microsecond=0).isoformat()
def _serialize_export_scholar(profile: ScholarProfile) -> dict[str, Any]:
@ -58,9 +58,7 @@ async def export_user_data(
user_id: int,
) -> dict[str, Any]:
scholars_result = await db_session.execute(
select(ScholarProfile)
.where(ScholarProfile.user_id == user_id)
.order_by(ScholarProfile.id.asc())
select(ScholarProfile).where(ScholarProfile.user_id == user_id).order_by(ScholarProfile.id.asc())
)
publication_result = await db_session.execute(
select(

4
app/services/domains/portability/normalize.py
View file

@ -104,6 +104,4 @@ def _validate_import_sizes(
if len(scholars) > MAX_IMPORT_SCHOLARS:
raise ImportExportError(f"Import exceeds max scholars ({MAX_IMPORT_SCHOLARS}).")
if len(publications) > MAX_IMPORT_PUBLICATIONS:
raise ImportExportError(
f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS})."
)
raise ImportExportError(f"Import exceeds max publications ({MAX_IMPORT_PUBLICATIONS}).")

10
app/services/domains/portability/publication_import.py
View file

@ -6,8 +6,8 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Publication, ScholarProfile, ScholarPublication
from app.services.domains.ingestion.application import build_publication_url, normalize_title
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.ingestion.application import build_publication_url, normalize_title
from app.services.domains.portability.normalize import (
_normalize_citation_count,
_normalize_optional_text,
@ -22,9 +22,7 @@ async def _find_publication_by_cluster(
*,
cluster_id: str,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(Publication.cluster_id == cluster_id)
)
result = await db_session.execute(select(Publication).where(Publication.cluster_id == cluster_id))
return result.scalar_one_or_none()
@ -33,9 +31,7 @@ async def _find_publication_by_fingerprint(
*,
fingerprint_sha256: str,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256)
)
result = await db_session.execute(select(Publication).where(Publication.fingerprint_sha256 == fingerprint_sha256))
return result.scalar_one_or_none()

8
app/services/domains/portability/scholar_import.py
View file

@ -15,9 +15,7 @@ async def _load_user_scholar_map(
*,
user_id: int,
) -> dict[str, ScholarProfile]:
result = await db_session.execute(
select(ScholarProfile).where(ScholarProfile.user_id == user_id)
)
result = await db_session.execute(select(ScholarProfile).where(ScholarProfile.user_id == user_id))
profiles = list(result.scalars().all())
return {profile.scholar_id: profile for profile in profiles}
@ -70,9 +68,7 @@ async def _upsert_imported_scholars(
for item in scholars:
try:
scholar_id = scholar_service.validate_scholar_id(str(item["scholar_id"]))
display_name = scholar_service.normalize_display_name(
str(item.get("display_name") or "")
)
display_name = scholar_service.normalize_display_name(str(item.get("display_name") or ""))
override_url = scholar_service.normalize_profile_image_url(
_normalize_optional_text(item.get("profile_image_override_url"))
)

4
app/services/domains/publication_identifiers/__init__.py
View file

@ -1,7 +1,7 @@
from app.services.domains.publication_identifiers.application import (
DisplayIdentifier,
display_identifier_for_publication_id,
derive_display_identifier_from_values,
display_identifier_for_publication_id,
overlay_pdf_queue_items_with_display_identifiers,
overlay_publication_items_with_display_identifiers,
sync_identifiers_for_publication_fields,
@ -10,8 +10,8 @@ from app.services.domains.publication_identifiers.application import (
__all__ = [
"DisplayIdentifier",
"display_identifier_for_publication_id",
"derive_display_identifier_from_values",
"display_identifier_for_publication_id",
"overlay_pdf_queue_items_with_display_identifiers",
"overlay_publication_items_with_display_identifiers",
"sync_identifiers_for_publication_fields",

18
app/services/domains/publication_identifiers/application.py
View file

@ -55,7 +55,9 @@ def _fallback_candidates_from_values(
if doi:
normalized_doi = normalize_doi(doi)
if normalized_doi:
candidates.append(_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url))
candidates.append(
_candidate(IdentifierKind.DOI, doi, normalized_doi, "legacy_doi", CONFIDENCE_HIGH, pub_url)
)
candidates.extend(_url_identifier_candidates(values=values, source="legacy_urls"))
return _dedup_candidates(candidates)
@ -332,10 +334,13 @@ async def _existing_identifier_by_kind(
kind: str,
) -> PublicationIdentifier | None:
result = await db_session.execute(
select(PublicationIdentifier).where(
select(PublicationIdentifier)
.where(
PublicationIdentifier.publication_id == publication_id,
PublicationIdentifier.kind == kind,
).order_by(PublicationIdentifier.confidence_score.desc()).limit(1)
)
.order_by(PublicationIdentifier.confidence_score.desc())
.limit(1)
)
return result.scalar_one_or_none()
@ -380,7 +385,9 @@ def _overlay_publication_item(
item: PublicationListItem,
display_identifier: DisplayIdentifier | None,
) -> PublicationListItem:
fallback = display_identifier or derive_display_identifier_from_values(doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url)
fallback = display_identifier or derive_display_identifier_from_values(
doi=None, pub_url=item.pub_url, pdf_url=item.pdf_url
)
return replace(item, display_identifier=fallback)
@ -447,8 +454,7 @@ def _best_display_identifier_map(rows: list[PublicationIdentifier]) -> dict[int,
return {
publication_id: display
for publication_id, display in (
(publication_id, _best_display_identifier(candidates))
for publication_id, candidates in grouped.items()
(publication_id, _best_display_identifier(candidates)) for publication_id, candidates in grouped.items()
)
if display is not None
}

60
app/services/domains/publications/application.py
View file

@ -1,21 +1,21 @@
from __future__ import annotations
from app.services.domains.publications.counts import (
count_for_user,
count_favorite_for_user,
count_for_user,
count_latest_for_user,
count_unread_for_user,
)
from app.services.domains.publications.listing import (
list_for_user,
retry_pdf_for_user,
list_unread_for_user,
)
from app.services.domains.publications.enrichment import (
hydrate_pdf_enrichment_state,
schedule_missing_pdf_enrichment_for_user,
schedule_retry_pdf_enrichment_for_row,
)
from app.services.domains.publications.listing import (
list_for_user,
list_unread_for_user,
retry_pdf_for_user,
)
from app.services.domains.publications.modes import (
MODE_ALL,
MODE_LATEST,
@ -23,6 +23,13 @@ from app.services.domains.publications.modes import (
MODE_UNREAD,
resolve_publication_view_mode,
)
from app.services.domains.publications.pdf_queue import (
count_pdf_queue_items,
enqueue_all_missing_pdf_jobs,
enqueue_retry_pdf_job_for_publication_id,
list_pdf_queue_items,
list_pdf_queue_page,
)
from app.services.domains.publications.queries import (
get_latest_run_id_for_user,
get_publication_item_for_user,
@ -33,42 +40,35 @@ from app.services.domains.publications.read_state import (
mark_selected_as_read_for_user,
set_publication_favorite_for_user,
)
from app.services.domains.publications.pdf_queue import (
count_pdf_queue_items,
enqueue_all_missing_pdf_jobs,
enqueue_retry_pdf_job_for_publication_id,
list_pdf_queue_page,
list_pdf_queue_items,
)
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem
__all__ = [
"MODE_ALL",
"MODE_UNREAD",
"MODE_LATEST",
"MODE_NEW",
"MODE_UNREAD",
"PublicationListItem",
"UnreadPublicationItem",
"resolve_publication_view_mode",
"get_latest_run_id_for_user",
"publications_query",
"get_publication_item_for_user",
"list_for_user",
"list_unread_for_user",
"retry_pdf_for_user",
"hydrate_pdf_enrichment_state",
"schedule_retry_pdf_enrichment_for_row",
"list_pdf_queue_items",
"list_pdf_queue_page",
"count_favorite_for_user",
"count_for_user",
"count_latest_for_user",
"count_pdf_queue_items",
"count_unread_for_user",
"enqueue_all_missing_pdf_jobs",
"enqueue_retry_pdf_job_for_publication_id",
"schedule_missing_pdf_enrichment_for_user",
"count_for_user",
"count_favorite_for_user",
"count_unread_for_user",
"count_latest_for_user",
"get_latest_run_id_for_user",
"get_publication_item_for_user",
"hydrate_pdf_enrichment_state",
"list_for_user",
"list_pdf_queue_items",
"list_pdf_queue_page",
"list_unread_for_user",
"mark_all_unread_as_read_for_user",
"mark_selected_as_read_for_user",
"publications_query",
"resolve_publication_view_mode",
"retry_pdf_for_user",
"schedule_missing_pdf_enrichment_for_user",
"schedule_retry_pdf_enrichment_for_row",
"set_publication_favorite_for_user",
]

18
app/services/domains/publications/dedup.py
View file

@ -1,9 +1,9 @@
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import logging
from typing import Iterable
from collections.abc import Iterable
from dataclasses import dataclass
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession
@ -113,9 +113,7 @@ async def _load_publication(
*,
publication_id: int,
) -> Publication | None:
result = await db_session.execute(
select(Publication).where(Publication.id == publication_id)
)
result = await db_session.execute(select(Publication).where(Publication.id == publication_id))
return result.scalar_one_or_none()
@ -160,9 +158,7 @@ async def _migrate_scholar_links(
dup_links = dup_links_result.scalars().all()
winner_profiles_result = await db_session.execute(
select(ScholarPublication.scholar_profile_id).where(
ScholarPublication.publication_id == winner_id
)
select(ScholarPublication.scholar_profile_id).where(ScholarPublication.publication_id == winner_id)
)
winner_profiles: set[int] = {row for (row,) in winner_profiles_result}
@ -346,11 +342,7 @@ def _candidate_from_row(
def _normalized_tokens(tokens: Iterable[str]) -> set[str]:
return {
token
for token in tokens
if len(token) >= NEAR_DUP_MIN_TOKEN_LENGTH and token not in NEAR_DUP_STOPWORDS
}
return {token for token in tokens if len(token) >= NEAR_DUP_MIN_TOKEN_LENGTH and token not in NEAR_DUP_STOPWORDS}
def _cluster_candidate_groups(

15
app/services/domains/publications/enrichment.py
View file

@ -4,12 +4,12 @@ import logging
from sqlalchemy.ext.asyncio import AsyncSession
from app.logging_utils import structured_log
from app.services.domains.publications.pdf_queue import (
enqueue_missing_pdf_jobs,
enqueue_retry_pdf_job,
overlay_pdf_job_state,
)
from app.logging_utils import structured_log
from app.services.domains.publications.types import PublicationListItem
logger = logging.getLogger(__name__)
@ -30,7 +30,9 @@ async def schedule_missing_pdf_enrichment_for_user(
rows=items,
max_items=max_items,
)
structured_log(logger, "info", "publications.enrichment.scheduled", user_id=user_id, publication_count=len(queued_ids))
structured_log(
logger, "info", "publications.enrichment.scheduled", user_id=user_id, publication_count=len(queued_ids)
)
return len(queued_ids)
@ -47,7 +49,14 @@ async def schedule_retry_pdf_enrichment_for_row(
request_email=request_email,
row=item,
)
structured_log(logger, "info", "publications.enrichment.retry_scheduled", user_id=user_id, publication_id=item.publication_id, queued=queued)
structured_log(
logger,
"info",
"publications.enrichment.retry_scheduled",
user_id=user_id,
publication_id=item.publication_id,
queued=queued,
)
return queued

5
app/services/domains/publications/listing.py
View file

@ -51,10 +51,7 @@ async def list_for_user(
snapshot_before=snapshot_before,
)
)
rows = [
publication_list_item_from_row(row, latest_run_id=latest_run_id)
for row in result.all()
]
rows = [publication_list_item_from_row(row, latest_run_id=latest_run_id) for row in result.all()]
return await identifier_service.overlay_publication_items_with_display_identifiers(
db_session,
items=rows,

45
app/services/domains/publications/pdf_queue.py
View file

@ -1,9 +1,9 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
import logging
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from sqlalchemy import Select, and_, func, literal, or_, select, union_all
from sqlalchemy.ext.asyncio import AsyncSession
@ -17,6 +17,7 @@ from app.db.models import (
User,
)
from app.db.session import get_session_factory
from app.logging_utils import structured_log
from app.services.domains.publication_identifiers import application as identifier_service
from app.services.domains.publication_identifiers.types import DisplayIdentifier
from app.services.domains.publications.pdf_resolution_pipeline import (
@ -27,7 +28,6 @@ from app.services.domains.unpaywall.application import (
FAILURE_RESOLUTION_EXCEPTION,
OaResolutionOutcome,
)
from app.logging_utils import structured_log
from app.settings import settings
PDF_STATUS_UNTRACKED = "untracked"
@ -85,7 +85,7 @@ class PdfQueuePage:
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
return datetime.now(UTC)
def _publication_ids(rows: list[PublicationListItem]) -> list[int]:
@ -472,7 +472,13 @@ async def _resolve_publication_row(
# Propagate upward so the batch loop can stop immediately.
raise
except Exception as exc: # pragma: no cover - defensive network boundary
structured_log(logger, "warning", "publications.pdf_queue.resolve_failed", publication_id=row.publication_id, error=str(exc))
structured_log(
logger,
"warning",
"publications.pdf_queue.resolve_failed",
publication_id=row.publication_id,
error=str(exc),
)
outcome = _failed_outcome(row=row)
arxiv_rate_limited = False
await _persist_outcome(
@ -514,9 +520,19 @@ async def _run_resolution_task(
)
if arxiv_rate_limited and arxiv_lookup_allowed:
arxiv_lookup_allowed = False
structured_log(logger, "warning", "pdf_queue.arxiv_batch_disabled", detail="arXiv temporarily disabled for remaining batch after rate limit")
structured_log(
logger,
"warning",
"pdf_queue.arxiv_batch_disabled",
detail="arXiv temporarily disabled for remaining batch after rate limit",
)
except OpenAlexBudgetExhaustedError:
structured_log(logger, "warning", "pdf_queue.budget_exhausted", detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted")
structured_log(
logger,
"warning",
"pdf_queue.budget_exhausted",
detail="Stopping PDF resolution batch — OpenAlex daily budget exhausted",
)
break
@ -679,7 +695,7 @@ async def _missing_pdf_candidates(
limit: int,
) -> list[PublicationListItem]:
bounded_limit = max(1, min(int(limit), 5000))
now = datetime.now(timezone.utc)
now = datetime.now(UTC)
cooldown_threshold = now - timedelta(days=7)
result = await db_session.execute(
@ -701,10 +717,7 @@ async def _missing_pdf_candidates(
.order_by(Publication.updated_at.desc(), Publication.id.desc())
.limit(bounded_limit)
)
return [
_queue_candidate_from_publication(publication)
for publication in result.scalars()
]
return [_queue_candidate_from_publication(publication) for publication in result.scalars()]
async def enqueue_all_missing_pdf_jobs(
@ -902,9 +915,7 @@ async def count_pdf_queue_items(
if normalized_status == PDF_STATUS_UNTRACKED:
result = await db_session.execute(_untracked_queue_count_select())
return int(result.scalar_one() or 0)
tracked_result = await db_session.execute(
_tracked_queue_count_select(status=normalized_status)
)
tracked_result = await db_session.execute(_tracked_queue_count_select(status=normalized_status))
tracked_count = int(tracked_result.scalar_one() or 0)
if normalized_status is not None:
return tracked_count
@ -946,9 +957,7 @@ async def drain_ready_jobs(
limit: int,
max_attempts: int,
) -> int:
result = await db_session.execute(
select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1)
)
result = await db_session.execute(select(User.id).where(User.is_active.is_(True)).order_by(User.id.asc()).limit(1))
system_user_id = result.scalar_one_or_none()
if system_user_id is None:
return 0

22
app/services/domains/publications/pdf_resolution_pipeline.py
View file

@ -1,15 +1,15 @@
from __future__ import annotations
from dataclasses import dataclass
import logging
from dataclasses import dataclass
from typing import Any
from app.logging_utils import structured_log
from app.services.domains.arxiv.application import ArxivRateLimitError
from app.services.domains.arxiv.guards import arxiv_skip_reason_for_item
from app.services.domains.openalex.client import OpenAlexBudgetExhaustedError
from app.services.domains.publications.types import PublicationListItem
from app.services.domains.unpaywall.application import OaResolutionOutcome, resolve_publication_oa_outcomes
from app.logging_utils import structured_log
from app.settings import settings
logger = logging.getLogger(__name__)
@ -66,6 +66,7 @@ async def _openalex_outcome(
return None
import re
safe_title = re.sub(r"[^\w\s]", " ", row.title)
safe_title = " ".join(safe_title.split())
if not safe_title:
@ -107,12 +108,24 @@ async def _arxiv_outcome(
from app.services.domains.arxiv.application import discover_arxiv_id_for_publication
if not allow_lookup:
structured_log(logger, "info", "pdf_resolution.arxiv_skipped", publication_id=int(row.publication_id), skip_reason="batch_arxiv_cooldown_active")
structured_log(
logger,
"info",
"pdf_resolution.arxiv_skipped",
publication_id=int(row.publication_id),
skip_reason="batch_arxiv_cooldown_active",
)
return None
skip_reason = arxiv_skip_reason_for_item(item=row)
if skip_reason is not None:
structured_log(logger, "info", "pdf_resolution.arxiv_skipped", publication_id=int(row.publication_id), skip_reason=skip_reason)
structured_log(
logger,
"info",
"pdf_resolution.arxiv_skipped",
publication_id=int(row.publication_id),
skip_reason=skip_reason,
)
return None
try:
@ -134,7 +147,6 @@ async def _arxiv_outcome(
return None
async def _oa_outcome(
*,
row: PublicationListItem,

4
app/services/domains/publications/queries.py
View file

@ -228,9 +228,7 @@ def publication_list_item_from_row(
is_read=bool(is_read),
is_favorite=bool(is_favorite),
first_seen_at=created_at,
is_new_in_latest_run=(
latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id
),
is_new_in_latest_run=(latest_run_id is not None and int(first_seen_run_id or 0) == latest_run_id),
)

6
app/services/domains/publications/read_state.py
View file

@ -17,11 +17,7 @@ def _normalized_selection_pairs(selections: list[tuple[int, int]]) -> set[tuple[
def _scoped_scholar_ids_query(*, user_id: int):
return (
select(ScholarProfile.id)
.where(ScholarProfile.user_id == user_id)
.scalar_subquery()
)
return select(ScholarProfile.id).where(ScholarProfile.user_id == user_id).scalar_subquery()
async def mark_all_unread_as_read_for_user(

18
app/services/domains/runs/application.py
View file

@ -25,21 +25,21 @@ from app.services.domains.runs.types import (
)
__all__ = [
"QUEUE_STATUS_DROPPED",
"QUEUE_STATUS_QUEUED",
"QUEUE_STATUS_RETRYING",
"QUEUE_STATUS_DROPPED",
"QueueListItem",
"QueueClearResult",
"QueueListItem",
"QueueTransitionError",
"clear_queue_item_for_user",
"drop_queue_item_for_user",
"extract_run_summary",
"get_manual_run_by_idempotency_key",
"get_queue_item_for_user",
"get_run_for_user",
"list_queue_items_for_user",
"list_recent_runs_for_user",
"list_runs_for_user",
"get_run_for_user",
"get_manual_run_by_idempotency_key",
"list_queue_items_for_user",
"get_queue_item_for_user",
"retry_queue_item_for_user",
"drop_queue_item_for_user",
"clear_queue_item_for_user",
"queue_status_counts_for_user",
"retry_queue_item_for_user",
]

17
app/services/domains/runs/events.py
View file

@ -1,14 +1,16 @@
import asyncio
import json
import logging
from typing import Any, AsyncGenerator, Dict, Set
from collections.abc import AsyncGenerator
from typing import Any
logger = logging.getLogger(__name__)
class RunEventPublisher:
def __init__(self) -> None:
# Maps run_id to a set of subscriber queues
self._subscribers: Dict[int, Set[asyncio.Queue]] = {}
self._subscribers: dict[int, set[asyncio.Queue]] = {}
def subscribe(self, run_id: int) -> asyncio.Queue:
if run_id not in self._subscribers:
@ -27,12 +29,9 @@ class RunEventPublisher:
async def publish(self, run_id: int, event_type: str, data: dict[str, Any]) -> None:
if run_id not in self._subscribers:
return
message = {
"type": event_type,
"data": data
}
message = {"type": event_type, "data": data}
# Fan-out to all active subscribers for this run
for queue in list(self._subscribers[run_id]):
try:
@ -40,8 +39,10 @@ class RunEventPublisher:
except asyncio.QueueFull:
logger.warning(f"Subscriber queue full for run {run_id}, dropping message")
run_events = RunEventPublisher()
async def event_generator(run_id: int) -> AsyncGenerator[str, None]:
queue = run_events.subscribe(run_id)
try:

4
app/services/domains/runs/queue_service.py
View file

@ -41,9 +41,7 @@ async def get_queue_item_for_user(
queue_item_id: int,
) -> QueueListItem | None:
result = await db_session.execute(
queue_item_select(user_id=user_id)
.where(IngestionQueueItem.id == queue_item_id)
.limit(1)
queue_item_select(user_id=user_id).where(IngestionQueueItem.id == queue_item_id).limit(1)
)
row = result.one_or_none()
if row is None:

4
app/services/domains/runs/runs_service.py
View file

@ -35,9 +35,7 @@ async def list_runs_for_user(
.limit(limit)
)
if failed_only:
stmt = stmt.where(
CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE])
)
stmt = stmt.where(CrawlRun.status.in_([RunStatus.FAILED, RunStatus.PARTIAL_FAILURE]))
result = await db_session.execute(stmt)
return list(result.scalars().all())

10
app/services/domains/runs/summary.py
View file

@ -24,9 +24,7 @@ def _summary_int_dict(summary: dict[str, Any], key: str) -> dict[str, int]:
if not isinstance(value, dict):
return {}
return {
str(item_key): _safe_int(item_value, 0)
for item_key, item_value in value.items()
if isinstance(item_key, str)
str(item_key): _safe_int(item_value, 0) for item_key, item_value in value.items() if isinstance(item_key, str)
}
@ -34,11 +32,7 @@ def _summary_bool_dict(summary: dict[str, Any], key: str) -> dict[str, bool]:
value = summary.get(key)
if not isinstance(value, dict):
return {}
return {
str(item_key): bool(item_value)
for item_key, item_value in value.items()
if isinstance(item_key, str)
}
return {str(item_key): bool(item_value) for item_key, item_value in value.items() if isinstance(item_key, str)}
def _retry_counts(summary: dict[str, Any]) -> dict[str, int]:

9
app/services/domains/scholar/author_rows.py
View file

@ -79,9 +79,7 @@ class ScholarAuthorSearchParser(HTMLParser):
return
affiliation = normalize_space("".join(self._candidate["aff_parts"])) or None
email_domain = _extract_verified_email_domain(
normalize_space("".join(self._candidate["eml_parts"])) or None
)
email_domain = _extract_verified_email_domain(normalize_space("".join(self._candidate["eml_parts"])) or None)
cited_by_text = normalize_space("".join(self._candidate["cby_parts"]))
cited_by_match = re.search(r"\d+", cited_by_text)
cited_by_count = int(cited_by_match.group(0)) if cited_by_match else None
@ -97,10 +95,7 @@ class ScholarAuthorSearchParser(HTMLParser):
profile_url = build_absolute_scholar_url(self._candidate["name_href"])
if not profile_url:
profile_url = (
"https://scholar.google.com/citations"
f"?hl=en&user={scholar_id}"
)
profile_url = f"https://scholar.google.com/citations?hl=en&user={scholar_id}"
self.candidates.append(
ScholarSearchCandidate(

11
app/services/domains/scholar/parser.py
View file

@ -3,40 +3,29 @@ from __future__ import annotations
from app.services.domains.scholar.author_rows import (
ScholarAuthorSearchParser,
count_author_search_markers,
parse_scholar_id_from_href,
)
from app.services.domains.scholar.parser_constants import SCRIPT_STYLE_RE
from app.services.domains.scholar.parser_types import (
ParseState,
ParsedAuthorSearchPage,
ParsedProfilePage,
PublicationCandidate,
ScholarDomInvariantError,
ScholarMalformedDataError,
ScholarParserError,
ScholarSearchCandidate,
)
from app.services.domains.scholar.parser_utils import (
strip_tags,
)
from app.services.domains.scholar.profile_rows import (
ScholarRowParser,
count_markers,
extract_articles_range,
extract_profile_image_url,
extract_profile_name,
extract_rows,
has_operation_error_banner,
has_show_more_button,
parse_citation_count,
parse_cluster_id_from_href,
parse_publications,
parse_year,
)
from app.services.domains.scholar.source import FetchResult
from app.services.domains.scholar.state_detection import (
classify_block_or_captcha_reason,
classify_network_error_reason,
detect_author_search_state,
detect_state,
)

3
app/services/domains/scholar/profile_rows.py
View file

@ -12,7 +12,6 @@ from app.services.domains.scholar.parser_types import PublicationCandidate
from app.services.domains.scholar.parser_utils import (
attr_class,
attr_href,
attr_src,
build_absolute_scholar_url,
normalize_space,
strip_tags,
@ -260,7 +259,7 @@ def has_show_more_button(html: str) -> bool:
def has_operation_error_banner(html: str) -> bool:
lowered = html.lower()
if "id=\"gsc_a_err\"" not in lowered and "id='gsc_a_err'" not in lowered:
if 'id="gsc_a_err"' not in lowered and "id='gsc_a_err'" not in lowered:
return False
return "can't perform the operation now" in lowered or "cannot perform the operation now" in lowered

31
app/services/domains/scholar/source.py
View file

@ -17,18 +17,12 @@ SCHOLAR_PROFILE_URL = "https://scholar.google.com/citations"
DEFAULT_PAGE_SIZE = 100
DEFAULT_USER_AGENTS = [
(
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
),
("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"),
(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 "
"(KHTML, like Gecko) Version/18.1 Safari/605.1.15"
),
(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) "
"Gecko/20100101 Firefox/131.0"
),
("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0"),
]
logger = logging.getLogger(__name__)
@ -44,8 +38,7 @@ class FetchResult:
class ScholarSource(Protocol):
async def fetch_profile_html(self, scholar_id: str) -> FetchResult:
...
async def fetch_profile_html(self, scholar_id: str) -> FetchResult: ...
async def fetch_profile_page_html(
self,
@ -53,16 +46,14 @@ class ScholarSource(Protocol):
*,
cstart: int,
pagesize: int,
) -> FetchResult:
...
) -> FetchResult: ...
async def fetch_author_search_html(
self,
query: str,
*,
start: int,
) -> FetchResult:
...
) -> FetchResult: ...
class LiveScholarSource:
@ -145,7 +136,9 @@ class LiveScholarSource:
@staticmethod
def _network_error_result(requested_url: str, exc: URLError) -> FetchResult:
structured_log(
logger, "warning", "scholar_source.fetch_network_error",
logger,
"warning",
"scholar_source.fetch_network_error",
requested_url=requested_url,
)
return FetchResult(
@ -159,7 +152,9 @@ class LiveScholarSource:
@staticmethod
def _http_error_result(requested_url: str, exc: HTTPError) -> FetchResult:
structured_log(
logger, "warning", "scholar_source.fetch_http_error",
logger,
"warning",
"scholar_source.fetch_http_error",
requested_url=requested_url,
status_code=exc.code,
)
@ -176,7 +171,9 @@ class LiveScholarSource:
body = response.read().decode("utf-8", errors="replace")
status_code = getattr(response, "status", 200)
structured_log(
logger, "debug", "scholar_source.fetch_succeeded",
logger,
"debug",
"scholar_source.fetch_succeeded",
requested_url=requested_url,
status_code=status_code,
)

137
app/services/domains/scholars/application.py
View file

@ -1,13 +1,11 @@
from __future__ import annotations
import asyncio
from dataclasses import replace
from datetime import datetime
from datetime import timedelta
from datetime import timezone
import logging
import os
import random
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from uuid import uuid4
from sqlalchemy import delete, func, select, text
@ -15,9 +13,10 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
from app.logging_utils import structured_log
from app.services.domains.scholar.parser import (
ParseState,
ParsedAuthorSearchPage,
ParseState,
ScholarParserError,
parse_author_search_page,
parse_profile_page,
@ -34,33 +33,30 @@ from app.services.domains.scholars.constants import (
DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
MAX_AUTHOR_SEARCH_LIMIT,
DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
MAX_AUTHOR_SEARCH_LIMIT,
SEARCH_CACHED_BLOCK_REASON,
SEARCH_COOLDOWN_REASON,
SEARCH_DISABLED_REASON,
)
from app.logging_utils import structured_log
from app.services.domains.scholars.exceptions import ScholarServiceError
from app.services.domains.scholars.search_hints import (
_merge_warnings,
_policy_blocked_author_search_result,
_trim_author_search_result,
resolve_profile_image,
scrape_state_hint,
)
from app.services.domains.scholars.uploads import (
_ensure_upload_root,
_resolve_upload_path,
_safe_remove_upload,
resolve_upload_file_path,
)
from app.services.domains.scholars.validators import (
normalize_display_name,
normalize_profile_image_url,
validate_scholar_id,
)
logger = logging.getLogger(__name__)
@ -247,7 +243,7 @@ async def _cache_get_author_search_result(
return None
expires_at = entry.expires_at
if expires_at.tzinfo is None:
expires_at = expires_at.replace(tzinfo=timezone.utc)
expires_at = expires_at.replace(tzinfo=UTC)
if expires_at <= now_utc:
await db_session.delete(entry)
return None
@ -305,27 +301,19 @@ async def _prune_author_search_cache(
now_utc: datetime,
max_entries: int,
) -> None:
await db_session.execute(
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc)
)
await db_session.execute(delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc))
bounded_max_entries = max(1, int(max_entries))
count_result = await db_session.execute(
select(func.count()).select_from(AuthorSearchCacheEntry)
)
count_result = await db_session.execute(select(func.count()).select_from(AuthorSearchCacheEntry))
entry_count = int(count_result.scalar_one() or 0)
overflow = max(0, entry_count - bounded_max_entries)
if overflow <= 0:
return
stale_keys_result = await db_session.execute(
select(AuthorSearchCacheEntry.query_key)
.order_by(AuthorSearchCacheEntry.cached_at.asc())
.limit(overflow)
select(AuthorSearchCacheEntry.query_key).order_by(AuthorSearchCacheEntry.cached_at.asc()).limit(overflow)
)
stale_keys = [str(row[0]) for row in stale_keys_result.all()]
if stale_keys:
await db_session.execute(
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys))
)
await db_session.execute(delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys)))
def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
@ -340,7 +328,7 @@ def _author_search_cooldown_remaining_seconds(
if cooldown_until is None:
return 0
if cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
cooldown_until = cooldown_until.replace(tzinfo=UTC)
remaining_seconds = int((cooldown_until - now_utc).total_seconds())
return max(0, remaining_seconds)
@ -432,7 +420,9 @@ def _normalize_author_search_inputs(query: str, limit: int) -> tuple[str, int, s
def _disabled_search_result(*, normalized_query: str, bounded_limit: int) -> ParsedAuthorSearchPage:
structured_log(
logger, "warning", "scholar_search.disabled_by_configuration",
logger,
"warning",
"scholar_search.disabled_by_configuration",
query=normalized_query,
)
return _policy_blocked_author_search_result(
@ -452,13 +442,15 @@ def _normalize_runtime_cooldown_state(
cooldown_until = runtime_state.cooldown_until
updated = False
if cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
cooldown_until = cooldown_until.replace(tzinfo=UTC)
runtime_state.cooldown_until = cooldown_until
updated = True
if now_utc < cooldown_until:
return updated
structured_log(
logger, "info", "scholar_search.cooldown_expired",
logger,
"info",
"scholar_search.cooldown_expired",
cooldown_until_utc=cooldown_until.isoformat(),
)
runtime_state.cooldown_until = None
@ -494,7 +486,9 @@ def _emit_cooldown_threshold_alert(
if bool(runtime_state.cooldown_alert_emitted):
return True
structured_log(
logger, "error", "scholar_search.cooldown_rejection_threshold_exceeded",
logger,
"error",
"scholar_search.cooldown_rejection_threshold_exceeded",
query=normalized_query,
cooldown_rejection_count=int(runtime_state.cooldown_rejection_count),
threshold=threshold,
@ -518,7 +512,9 @@ def _cooldown_block_result(
cooldown_rejection_alert_threshold=cooldown_rejection_alert_threshold,
)
structured_log(
logger, "warning", "scholar_search.cooldown_active",
logger,
"warning",
"scholar_search.cooldown_active",
query=normalized_query,
cooldown_remaining_seconds=cooldown_remaining_seconds,
cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
@ -549,7 +545,9 @@ async def _cache_hit_result(
if cached is None:
return None
structured_log(
logger, "info", "scholar_search.cache_hit",
logger,
"info",
"scholar_search.cache_hit",
query=normalized_query,
state=cached.state.value,
state_reason=cached.state_reason,
@ -576,7 +574,7 @@ def _throttle_sleep_seconds(
else:
last_live_request_at = runtime_state.last_live_request_at
if last_live_request_at.tzinfo is None:
last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc)
last_live_request_at = last_live_request_at.replace(tzinfo=UTC)
runtime_state.last_live_request_at = last_live_request_at
updated = True
enforced_wait_seconds = (
@ -603,7 +601,9 @@ async def _wait_for_author_search_throttle(
if sleep_seconds <= 0.0:
return updated
structured_log(
logger, "info", "scholar_search.throttle_wait",
logger,
"info",
"scholar_search.throttle_wait",
query=normalized_query,
sleep_seconds=round(sleep_seconds, 3),
)
@ -659,7 +659,9 @@ def _with_retry_warnings(
if retry_scheduled_count < threshold:
return merged
structured_log(
logger, "warning", "scholar_search.retry_threshold_exceeded",
logger,
"warning",
"scholar_search.retry_threshold_exceeded",
query=normalized_query,
retry_scheduled_count=retry_scheduled_count,
threshold=threshold,
@ -688,19 +690,23 @@ def _apply_block_circuit_breaker(
return merged_parsed
runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
structured_log(
logger, "warning", "scholar_search.block_detected",
logger,
"warning",
"scholar_search.block_detected",
query=normalized_query,
state_reason=merged_parsed.state_reason,
consecutive_blocked_count=int(runtime_state.consecutive_blocked_count),
)
if int(runtime_state.consecutive_blocked_count) < max(1, int(cooldown_block_threshold)):
return merged_parsed
runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(seconds=max(60, int(cooldown_seconds)))
runtime_state.cooldown_until = datetime.now(UTC) + timedelta(seconds=max(60, int(cooldown_seconds)))
runtime_state.consecutive_blocked_count = 0
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
structured_log(
logger, "error", "scholar_search.cooldown_activated",
logger,
"error",
"scholar_search.cooldown_activated",
query=normalized_query,
cooldown_until_utc=runtime_state.cooldown_until.isoformat() if runtime_state.cooldown_until else None,
)
@ -739,11 +745,11 @@ async def _cooldown_or_cache_result(
) -> tuple[ParsedAuthorSearchPage | None, bool]:
runtime_state_updated = _normalize_runtime_cooldown_state(
runtime_state,
now_utc=datetime.now(timezone.utc),
now_utc=datetime.now(UTC),
)
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(
runtime_state,
datetime.now(timezone.utc),
datetime.now(UTC),
)
if cooldown_remaining_seconds > 0:
return (
@ -759,18 +765,35 @@ async def _cooldown_or_cache_result(
cached_result = await _cache_hit_result(
db_session,
query_key=query_key,
now_utc=datetime.now(timezone.utc),
now_utc=datetime.now(UTC),
normalized_query=normalized_query,
bounded_limit=bounded_limit,
)
return cached_result, runtime_state_updated
async def _perform_live_author_search(db_session: AsyncSession, *, source: ScholarSource, runtime_state: AuthorSearchRuntimeState, normalized_query: str, query_key: str, network_error_retries: int, retry_backoff_seconds: float, min_interval_seconds: float, interval_jitter_seconds: float, retry_alert_threshold: int, cooldown_block_threshold: int, cooldown_seconds: int, blocked_cache_ttl_seconds: int, cache_ttl_seconds: int, cache_max_entries: int) -> tuple[ParsedAuthorSearchPage, bool]:
async def _perform_live_author_search(
db_session: AsyncSession,
*,
source: ScholarSource,
runtime_state: AuthorSearchRuntimeState,
normalized_query: str,
query_key: str,
network_error_retries: int,
retry_backoff_seconds: float,
min_interval_seconds: float,
interval_jitter_seconds: float,
retry_alert_threshold: int,
cooldown_block_threshold: int,
cooldown_seconds: int,
blocked_cache_ttl_seconds: int,
cache_ttl_seconds: int,
cache_max_entries: int,
) -> tuple[ParsedAuthorSearchPage, bool]:
runtime_state_updated = await _wait_for_author_search_throttle(
runtime_state=runtime_state,
normalized_query=normalized_query,
now_utc=datetime.now(timezone.utc),
now_utc=datetime.now(UTC),
min_interval_seconds=min_interval_seconds,
interval_jitter_seconds=interval_jitter_seconds,
)
@ -780,7 +803,7 @@ async def _perform_live_author_search(db_session: AsyncSession, *, source: Schol
network_error_retries=network_error_retries,
retry_backoff_seconds=retry_backoff_seconds,
)
runtime_state.last_live_request_at = datetime.now(timezone.utc)
runtime_state.last_live_request_at = datetime.now(UTC)
merged = _with_retry_warnings(
parsed,
retry_warnings=retry_warnings,
@ -806,12 +829,30 @@ async def _perform_live_author_search(db_session: AsyncSession, *, source: Schol
parsed=merged,
ttl_seconds=float(ttl_seconds),
max_entries=cache_max_entries,
now_utc=datetime.now(timezone.utc),
now_utc=datetime.now(UTC),
)
return merged, True
async def search_author_candidates(*, source: ScholarSource, db_session: AsyncSession, query: str, limit: int, network_error_retries: int = 1, retry_backoff_seconds: float = 1.0, search_enabled: bool = True, cache_ttl_seconds: int = 21_600, blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS, cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES, min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS, interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS, cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD, cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD) -> ParsedAuthorSearchPage:
async def search_author_candidates(
*,
source: ScholarSource,
db_session: AsyncSession,
query: str,
limit: int,
network_error_retries: int = 1,
retry_backoff_seconds: float = 1.0,
search_enabled: bool = True,
cache_ttl_seconds: int = 21_600,
blocked_cache_ttl_seconds: int = DEFAULT_AUTHOR_SEARCH_BLOCKED_CACHE_TTL_SECONDS,
cache_max_entries: int = DEFAULT_AUTHOR_SEARCH_CACHE_MAX_ENTRIES,
min_interval_seconds: float = DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS,
interval_jitter_seconds: float = DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS,
cooldown_block_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_BLOCK_THRESHOLD,
cooldown_seconds: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_SECONDS,
retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
) -> ParsedAuthorSearchPage:
normalized_query, bounded_limit, query_key = _normalize_author_search_inputs(query, limit)
if not search_enabled:
return _disabled_search_result(
@ -924,17 +965,13 @@ async def set_profile_image_upload(
normalized_content_type = (content_type or "").strip().lower()
extension = ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES.get(normalized_content_type)
if extension is None:
raise ScholarServiceError(
"Unsupported image type. Use JPEG, PNG, WEBP, or GIF."
)
raise ScholarServiceError("Unsupported image type. Use JPEG, PNG, WEBP, or GIF.")
if not image_bytes:
raise ScholarServiceError("Uploaded image file is empty.")
if len(image_bytes) > max_upload_bytes:
raise ScholarServiceError(
f"Uploaded image exceeds {max_upload_bytes} bytes."
)
raise ScholarServiceError(f"Uploaded image exceeds {max_upload_bytes} bytes.")
upload_root = _ensure_upload_root(upload_dir, create=True)
user_dir = upload_root / str(profile.user_id)

30
app/services/domains/scholars/constants.py
View file

@ -32,39 +32,25 @@ SEARCH_CACHED_BLOCK_REASON = "search_temporarily_disabled_from_cached_blocked_re
STATE_REASON_HINTS: dict[str, str] = {
SEARCH_DISABLED_REASON: (
"Scholar name search is currently disabled by service policy. "
"Add scholars by profile URL or Scholar ID."
"Scholar name search is currently disabled by service policy. Add scholars by profile URL or Scholar ID."
),
SEARCH_COOLDOWN_REASON: (
"Scholar name search is temporarily paused after repeated block responses. "
"Use Scholar URL/ID adds until cooldown expires."
),
SEARCH_CACHED_BLOCK_REASON: (
"A recent blocked response was cached to reduce traffic. "
"Retry later or add by Scholar URL/ID."
"A recent blocked response was cached to reduce traffic. Retry later or add by Scholar URL/ID."
),
"network_dns_resolution_failed": (
"DNS resolution failed while reaching scholar.google.com. "
"Verify container DNS/network and retry."
),
"network_timeout": (
"Request timed out before Google Scholar responded. "
"Increase delay/backoff and retry."
),
"network_tls_error": (
"TLS handshake/certificate validation failed. "
"Verify outbound TLS/network configuration."
),
"blocked_http_429_rate_limited": (
"Google Scholar rate-limited the request. "
"Slow request cadence and retry later."
"DNS resolution failed while reaching scholar.google.com. Verify container DNS/network and retry."
),
"network_timeout": ("Request timed out before Google Scholar responded. Increase delay/backoff and retry."),
"network_tls_error": ("TLS handshake/certificate validation failed. Verify outbound TLS/network configuration."),
"blocked_http_429_rate_limited": ("Google Scholar rate-limited the request. Slow request cadence and retry later."),
"blocked_unusual_traffic_detected": (
"Google Scholar flagged traffic as unusual. "
"Increase delay/jitter and reduce concurrent scraping."
"Google Scholar flagged traffic as unusual. Increase delay/jitter and reduce concurrent scraping."
),
"blocked_accounts_redirect": (
"Request was redirected to Google Account sign-in. "
"Treat as access block and retry later."
"Request was redirected to Google Account sign-in. Treat as access block and retry later."
),
}

2
app/services/domains/scholars/search_hints.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
from app.db.models import ScholarProfile
from app.services.domains.scholar.parser import ParseState, ParsedAuthorSearchPage
from app.services.domains.scholar.parser import ParsedAuthorSearchPage, ParseState
from app.services.domains.scholars.constants import (
MAX_AUTHOR_SEARCH_LIMIT,
STATE_REASON_HINTS,

4
app/services/domains/scholars/validators.py
View file

@ -27,9 +27,7 @@ def normalize_profile_image_url(value: str | None) -> str | None:
return None
if len(candidate) > MAX_IMAGE_URL_LENGTH:
raise ScholarServiceError(
f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer."
)
raise ScholarServiceError(f"Image URL must be {MAX_IMAGE_URL_LENGTH} characters or fewer.")
parsed = urlparse(candidate)
if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:

17
app/services/domains/settings/application.py
View file

@ -60,9 +60,7 @@ def parse_run_interval_minutes(value: str, *, minimum: int = HARD_MIN_RUN_INTERV
raise UserSettingsServiceError("Check interval must be a whole number.") from exc
effective_minimum = resolve_run_interval_minimum(minimum)
if parsed < effective_minimum:
raise UserSettingsServiceError(
f"Check interval must be at least {effective_minimum} minutes."
)
raise UserSettingsServiceError(f"Check interval must be at least {effective_minimum} minutes.")
return parsed
@ -73,9 +71,7 @@ def parse_request_delay_seconds(value: str, *, minimum: int = HARD_MIN_REQUEST_D
raise UserSettingsServiceError("Request delay must be a whole number.") from exc
effective_minimum = resolve_request_delay_minimum(minimum)
if parsed < effective_minimum:
raise UserSettingsServiceError(
f"Request delay must be at least {effective_minimum} seconds."
)
raise UserSettingsServiceError(f"Request delay must be at least {effective_minimum} seconds.")
return parsed
@ -102,23 +98,20 @@ def parse_nav_visible_pages(value: object) -> list[str]:
missing_required = [page for page in REQUIRED_NAV_PAGES if page not in seen]
if missing_required:
raise UserSettingsServiceError(
"Dashboard, Scholars, and Settings must remain visible."
)
raise UserSettingsServiceError("Dashboard, Scholars, and Settings must remain visible.")
return deduped
from app.settings import settings as app_settings
async def get_or_create_settings(
db_session: AsyncSession,
*,
user_id: int,
) -> UserSetting:
result = await db_session.execute(
select(UserSetting).where(UserSetting.user_id == user_id)
)
result = await db_session.execute(select(UserSetting).where(UserSetting.user_id == user_id))
settings = result.scalar_one_or_none()
if settings is not None:
return settings

24
app/services/domains/unpaywall/application.py
View file

@ -1,11 +1,12 @@
from __future__ import annotations
from dataclasses import dataclass
import logging
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING
from urllib.parse import unquote
from app.logging_utils import structured_log
from app.services.domains.crossref.application import discover_doi_for_publication
from app.services.domains.doi.normalize import normalize_doi
from app.services.domains.unpaywall.pdf_discovery import (
@ -13,7 +14,6 @@ from app.services.domains.unpaywall.pdf_discovery import (
resolve_pdf_from_landing_page,
)
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot
from app.logging_utils import structured_log
from app.settings import settings
if TYPE_CHECKING:
@ -67,11 +67,8 @@ def _publication_doi(item: PublicationListItem | UnreadPublicationItem) -> str |
stored = None
if getattr(item, "display_identifier", None) and item.display_identifier.kind == "doi":
stored = normalize_doi(item.display_identifier.value)
explicit_doi = (
_extract_explicit_doi(item.pub_url)
or _extract_explicit_doi(item.venue_text)
)
explicit_doi = _extract_explicit_doi(item.pub_url) or _extract_explicit_doi(item.venue_text)
if explicit_doi:
return explicit_doi
pub_url_doi = _extract_doi_candidate(item.pub_url)
@ -289,10 +286,7 @@ def _resolved_pdf_count(outcomes: dict[int, OaResolutionOutcome]) -> int:
def _outcome_metadata(outcomes: dict[int, OaResolutionOutcome]) -> dict[int, tuple[str | None, str | None]]:
return {
publication_id: (outcome.doi, outcome.pdf_url)
for publication_id, outcome in outcomes.items()
}
return {publication_id: (outcome.doi, outcome.pdf_url) for publication_id, outcome in outcomes.items()}
async def _resolve_outcome_for_item(
@ -347,7 +341,9 @@ async def _safe_outcome_for_item(
allow_crossref=allow_crossref,
)
except Exception as exc: # pragma: no cover - defensive network boundary
structured_log(logger, "warning", "unpaywall.resolve_item_failed", publication_id=item.publication_id, error=str(exc))
structured_log(
logger, "warning", "unpaywall.resolve_item_failed", publication_id=item.publication_id, error=str(exc)
)
return _outcome_with_failure(
item=item,
failure_reason=FAILURE_RESOLUTION_EXCEPTION,
@ -410,7 +406,9 @@ async def resolve_publication_oa_outcomes(
email=email,
)
structured_log(
logger, "info", "unpaywall.resolve_completed",
logger,
"info",
"unpaywall.resolve_completed",
publication_count=len(items),
doi_input_count=_doi_input_count(items),
search_attempt_count=_search_attempt_count(targets=targets),

2
app/services/domains/unpaywall/pdf_discovery.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
from html.parser import HTMLParser
import re
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
from app.services.domains.unpaywall.rate_limit import wait_for_unpaywall_slot

5
app/services/domains/users/application.py
View file

@ -39,9 +39,7 @@ async def get_user_by_id(db_session: AsyncSession, user_id: int) -> User | None:
async def get_user_by_email(db_session: AsyncSession, email: str) -> User | None:
result = await db_session.execute(
select(User).where(User.email == normalize_email(email))
)
result = await db_session.execute(select(User).where(User.email == normalize_email(email)))
return result.scalar_one_or_none()
@ -95,4 +93,3 @@ async def set_user_password_hash(
await db_session.commit()
await db_session.refresh(user)
return user