justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

remove legacy in-memory search state fallback

Browse source
This commit is contained in:
Justin Visser 2026-02-19 22:31:26 +01:00
parent d7404abf18
commit 1ce85304e3
17 changed files with 766 additions and 248 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app/api/routers/scholars.py
View file

@ -149,12 +149,14 @@ async def search_scholars(
request: Request,
query: str = Query(..., min_length=2, max_length=120),
limit: int = Query(10, ge=1, le=25),
db_session: AsyncSession = Depends(get_db_session),
source: ScholarSource = Depends(get_scholar_source),
current_user: User = Depends(get_api_current_user),
):
try:
parsed = await scholar_service.search_author_candidates(
source=source,
db_session=db_session,
query=query,
limit=limit,
network_error_retries=settings.ingestion_network_error_retries,

17
app/auth/rate_limit.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from collections import defaultdict, deque
from collections import deque
from collections.abc import Callable
from dataclasses import dataclass
from math import ceil
@ -25,14 +25,19 @@ class SlidingWindowRateLimiter:
self._max_attempts = max_attempts
self._window_seconds = window_seconds
self._now = now
self._attempts: dict[str, deque[float]] = defaultdict(deque)
self._attempts: dict[str, deque[float]] = {}
self._lock = Lock()
def check(self, key: str) -> RateLimitDecision:
with self._lock:
now_value = self._now()
attempts = self._attempts[key]
attempts = self._attempts.get(key)
if attempts is None:
return RateLimitDecision(allowed=True)
self._trim_expired(attempts, now_value)
if not attempts:
self._attempts.pop(key, None)
return RateLimitDecision(allowed=True)
if len(attempts) >= self._max_attempts:
retry_after = self._window_seconds - (now_value - attempts[0])
return RateLimitDecision(
@ -44,7 +49,10 @@ class SlidingWindowRateLimiter:
def record_failure(self, key: str) -> None:
with self._lock:
now_value = self._now()
attempts = self._attempts[key]
attempts = self._attempts.get(key)
if attempts is None:
attempts = deque()
self._attempts[key] = attempts
self._trim_expired(attempts, now_value)
attempts.append(now_value)
@ -59,4 +67,3 @@ class SlidingWindowRateLimiter:
def _trim_expired(self, attempts: deque[float], now_value: float) -> None:
while attempts and now_value - attempts[0] >= self._window_seconds:
attempts.popleft()

53
app/db/models.py
View file

@ -319,3 +319,56 @@ class IngestionQueueItem(Base):
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class AuthorSearchRuntimeState(Base):
__tablename__ = "author_search_runtime_state"
state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
consecutive_blocked_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
cooldown_rejection_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
cooldown_alert_emitted: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
server_default=text("false"),
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class AuthorSearchCacheEntry(Base):
__tablename__ = "author_search_cache_entries"
__table_args__ = (
Index("ix_author_search_cache_expires_at", "expires_at"),
Index("ix_author_search_cache_cached_at", "cached_at"),
)
query_key: Mapped[str] = mapped_column(String(256), primary_key=True)
payload: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
)
expires_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
)
cached_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)

47
app/db/session.py
View file

@ -1,5 +1,6 @@
from collections.abc import AsyncIterator
import logging
import os
from sqlalchemy import text
from sqlalchemy.ext.asyncio import (
@ -18,16 +19,50 @@ _engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None
def _normalized_pool_mode(raw_mode: str) -> str:
mode = (raw_mode or "").strip().lower()
if mode == "auto":
if os.getenv("PYTEST_CURRENT_TEST"):
return "null"
app_env = (os.getenv("APP_ENV") or "").strip().lower()
if app_env in {"test", "development", "dev", "local"}:
return "null"
return "queue"
if mode in {"null", "queue"}:
return mode
logger.warning(
"db.invalid_pool_mode_fallback",
extra={
"event": "db.invalid_pool_mode_fallback",
"database_pool_mode": raw_mode,
"fallback_mode": "queue",
},
)
return "queue"
def get_engine() -> AsyncEngine:
global _engine
if _engine is None:
# NullPool avoids cross-event-loop connection reuse in tests and dev tools.
_engine = create_async_engine(
settings.database_url,
pool_pre_ping=True,
poolclass=NullPool,
pool_mode = _normalized_pool_mode(settings.database_pool_mode)
engine_kwargs: dict[str, object] = {
"pool_pre_ping": True,
}
if pool_mode == "null":
engine_kwargs["poolclass"] = NullPool
else:
engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size))
engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow))
engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
_engine = create_async_engine(settings.database_url, **engine_kwargs)
logger.info(
"db.engine_initialized",
extra={
"event": "db.engine_initialized",
"pool_mode": pool_mode,
},
)
logger.info("db.engine_initialized", extra={"event": "db.engine_initialized"})
return _engine

2
app/services/scholar_source.py
View file

@ -195,7 +195,7 @@ def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
if cstart > 0:
query["cstart"] = int(cstart)
if pagesize > 0 and cstart > 0:
if pagesize > 0:
query["pagesize"] = int(pagesize)
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"

643
app/services/scholars.py
View file

@ -1,8 +1,6 @@
from __future__ import annotations
import asyncio
from collections import OrderedDict
from dataclasses import dataclass
from dataclasses import replace
from datetime import datetime
from datetime import timedelta
@ -11,16 +9,15 @@ import logging
import os
import random
import re
import time
from pathlib import Path
from urllib.parse import urlparse
from uuid import uuid4
from sqlalchemy import select
from sqlalchemy import delete, func, select, text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ScholarProfile
from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
from app.services.scholar_parser import (
ParseState,
ParsedAuthorSearchPage,
@ -40,6 +37,9 @@ DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2
DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3
AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global"
AUTHOR_SEARCH_LOCK_NAMESPACE = 3901
AUTHOR_SEARCH_LOCK_KEY = 1
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
"image/jpeg": ".jpg",
"image/png": ".png",
@ -91,22 +91,6 @@ STATE_REASON_HINTS: dict[str, str] = {
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class _AuthorSearchCacheEntry:
parsed: ParsedAuthorSearchPage
expires_at_monotonic: float
cached_at_utc: datetime
_AUTHOR_SEARCH_EXECUTION_LOCK = asyncio.Lock()
_AUTHOR_SEARCH_CACHE: OrderedDict[str, _AuthorSearchCacheEntry] = OrderedDict()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC: datetime | None = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
class ScholarServiceError(ValueError):
"""Raised for expected scholar-management validation failures."""
@ -235,64 +219,260 @@ def _policy_blocked_author_search_result(
)
def _cache_get_author_search_result(query_key: str, now_monotonic: float) -> _AuthorSearchCacheEntry | None:
entry = _AUTHOR_SEARCH_CACHE.get(query_key)
async def _acquire_author_search_lock(db_session: AsyncSession) -> None:
await db_session.execute(
text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
{
"namespace": AUTHOR_SEARCH_LOCK_NAMESPACE,
"lock_key": AUTHOR_SEARCH_LOCK_KEY,
},
)
async def _load_runtime_state_for_update(
db_session: AsyncSession,
) -> AuthorSearchRuntimeState:
result = await db_session.execute(
select(AuthorSearchRuntimeState)
.where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY)
.with_for_update()
)
state = result.scalar_one_or_none()
if state is not None:
return state
state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY)
db_session.add(state)
await db_session.flush()
return state
def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict:
return {
"state": parsed.state.value,
"state_reason": parsed.state_reason,
"marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()},
"warnings": [str(value) for value in parsed.warnings],
"candidates": [
{
"scholar_id": candidate.scholar_id,
"display_name": candidate.display_name,
"affiliation": candidate.affiliation,
"email_domain": candidate.email_domain,
"cited_by_count": candidate.cited_by_count,
"interests": [str(interest) for interest in candidate.interests],
"profile_url": candidate.profile_url,
"profile_image_url": candidate.profile_image_url,
}
for candidate in parsed.candidates
],
}
def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
if not isinstance(payload, dict):
return None
state_raw = str(payload.get("state", "")).strip()
try:
state = ParseState(state_raw)
except ValueError:
return None
marker_counts_payload = payload.get("marker_counts")
marker_counts = (
{str(key): int(value) for key, value in marker_counts_payload.items()}
if isinstance(marker_counts_payload, dict)
else {}
)
warnings_payload = payload.get("warnings")
warnings = (
[str(value) for value in warnings_payload if isinstance(value, str)]
if isinstance(warnings_payload, list)
else []
)
from app.services.scholar_parser import ScholarSearchCandidate
candidates_payload = payload.get("candidates")
normalized_candidates = []
for value in candidates_payload if isinstance(candidates_payload, list) else []:
if not isinstance(value, dict):
continue
scholar_id = str(value.get("scholar_id", "")).strip()
display_name = str(value.get("display_name", "")).strip()
profile_url = str(value.get("profile_url", "")).strip()
if not scholar_id or not display_name or not profile_url:
continue
interests_payload = value.get("interests")
interests = (
[str(item) for item in interests_payload if isinstance(item, str)]
if isinstance(interests_payload, list)
else []
)
cited_by_count = value.get("cited_by_count")
parsed_cited_by_count: int | None = None
if isinstance(cited_by_count, int):
parsed_cited_by_count = cited_by_count
elif isinstance(cited_by_count, str) and cited_by_count.strip():
try:
parsed_cited_by_count = int(cited_by_count)
except ValueError:
parsed_cited_by_count = None
normalized_candidates.append(
{
"scholar_id": scholar_id,
"display_name": display_name,
"affiliation": str(value.get("affiliation")).strip()
if value.get("affiliation") is not None
else None,
"email_domain": str(value.get("email_domain")).strip()
if value.get("email_domain") is not None
else None,
"cited_by_count": parsed_cited_by_count,
"interests": interests,
"profile_url": profile_url,
"profile_image_url": str(value.get("profile_image_url")).strip()
if value.get("profile_image_url") is not None
else None,
}
)
return ParsedAuthorSearchPage(
state=state,
state_reason=str(payload.get("state_reason", "")).strip() or "unknown",
candidates=[
ScholarSearchCandidate(
scholar_id=item["scholar_id"],
display_name=item["display_name"],
affiliation=item["affiliation"],
email_domain=item["email_domain"],
cited_by_count=item["cited_by_count"],
interests=item["interests"],
profile_url=item["profile_url"],
profile_image_url=item["profile_image_url"],
)
for item in normalized_candidates
],
marker_counts=marker_counts,
warnings=warnings,
)
async def _cache_get_author_search_result(
db_session: AsyncSession,
*,
query_key: str,
now_utc: datetime,
) -> ParsedAuthorSearchPage | None:
result = await db_session.execute(
select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
)
entry = result.scalar_one_or_none()
if entry is None:
return None
if entry.expires_at_monotonic <= now_monotonic:
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
expires_at = entry.expires_at
if expires_at.tzinfo is None:
expires_at = expires_at.replace(tzinfo=timezone.utc)
if expires_at <= now_utc:
await db_session.delete(entry)
return None
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
return entry
parsed = _deserialize_parsed_author_search_page(entry.payload)
if parsed is None:
await db_session.delete(entry)
return None
return parsed
def _cache_set_author_search_result(
async def _cache_set_author_search_result(
db_session: AsyncSession,
*,
query_key: str,
parsed: ParsedAuthorSearchPage,
ttl_seconds: float,
max_entries: int,
now_utc: datetime,
) -> None:
ttl = max(float(ttl_seconds), 0.0)
existing_result = await db_session.execute(
select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
)
existing = existing_result.scalar_one_or_none()
if ttl <= 0.0:
_AUTHOR_SEARCH_CACHE.pop(query_key, None)
if existing is not None:
await db_session.delete(existing)
return
_AUTHOR_SEARCH_CACHE[query_key] = _AuthorSearchCacheEntry(
parsed=parsed,
expires_at_monotonic=time.monotonic() + ttl,
cached_at_utc=datetime.now(timezone.utc),
)
_AUTHOR_SEARCH_CACHE.move_to_end(query_key)
expires_at = now_utc + timedelta(seconds=ttl)
payload = _serialize_parsed_author_search_page(parsed)
if existing is None:
db_session.add(
AuthorSearchCacheEntry(
query_key=query_key,
payload=payload,
expires_at=expires_at,
cached_at=now_utc,
updated_at=now_utc,
)
)
else:
existing.payload = payload
existing.expires_at = expires_at
existing.cached_at = now_utc
existing.updated_at = now_utc
await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries)
async def _prune_author_search_cache(
db_session: AsyncSession,
*,
now_utc: datetime,
max_entries: int,
) -> None:
await db_session.execute(
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc)
)
bounded_max_entries = max(1, int(max_entries))
while len(_AUTHOR_SEARCH_CACHE) > bounded_max_entries:
_AUTHOR_SEARCH_CACHE.popitem(last=False)
count_result = await db_session.execute(
select(func.count()).select_from(AuthorSearchCacheEntry)
)
entry_count = int(count_result.scalar_one() or 0)
overflow = max(0, entry_count - bounded_max_entries)
if overflow <= 0:
return
stale_keys_result = await db_session.execute(
select(AuthorSearchCacheEntry.query_key)
.order_by(AuthorSearchCacheEntry.cached_at.asc())
.limit(overflow)
)
stale_keys = [str(row[0]) for row in stale_keys_result.all()]
if stale_keys:
await db_session.execute(
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys))
)
def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
def _author_search_cooldown_remaining_seconds(now_utc: datetime) -> int:
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is None:
def _author_search_cooldown_remaining_seconds(
runtime_state: AuthorSearchRuntimeState,
now_utc: datetime,
) -> int:
cooldown_until = runtime_state.cooldown_until
if cooldown_until is None:
return 0
remaining_seconds = int((_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC - now_utc).total_seconds())
if cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
remaining_seconds = int((cooldown_until - now_utc).total_seconds())
return max(0, remaining_seconds)
def _reset_author_search_runtime_state_for_tests() -> None:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
global _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
global _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
_AUTHOR_SEARCH_CACHE.clear()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
# Runtime state now lives in the database; tests should reset via DB fixtures.
return None
async def list_scholars_for_user(
@ -375,6 +555,7 @@ async def delete_scholar(
async def search_author_candidates(
*,
source: ScholarSource,
db_session: AsyncSession,
query: str,
limit: int,
network_error_retries: int = 1,
@ -390,12 +571,6 @@ async def search_author_candidates(
retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
) -> ParsedAuthorSearchPage:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
global _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
global _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
normalized_query = query.strip()
if len(normalized_query) < 2:
raise ScholarServiceError("Search query must be at least 2 characters.")
@ -416,208 +591,234 @@ async def search_author_candidates(
limit=bounded_limit,
)
async with _AUTHOR_SEARCH_EXECUTION_LOCK:
now_utc = datetime.now(timezone.utc)
now_monotonic = time.monotonic()
await _acquire_author_search_lock(db_session)
runtime_state = await _load_runtime_state_for_update(db_session)
runtime_state_updated = False
now_utc = datetime.now(timezone.utc)
if (
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is not None
and now_utc >= _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
):
if runtime_state.cooldown_until is not None:
cooldown_until = runtime_state.cooldown_until
if cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
runtime_state.cooldown_until = cooldown_until
runtime_state_updated = True
if now_utc >= cooldown_until:
logger.info(
"scholar_search.cooldown_expired",
extra={
"event": "scholar_search.cooldown_expired",
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(),
"cooldown_until_utc": cooldown_until.isoformat(),
},
)
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
runtime_state.cooldown_until = None
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
runtime_state_updated = True
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(now_utc)
if cooldown_remaining_seconds > 0:
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT += 1
bounded_cooldown_rejection_alert_threshold = max(
1,
int(cooldown_rejection_alert_threshold),
)
if (
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
>= bounded_cooldown_rejection_alert_threshold
and not _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
):
logger.error(
"scholar_search.cooldown_rejection_threshold_exceeded",
extra={
"event": "scholar_search.cooldown_rejection_threshold_exceeded",
"query": normalized_query,
"cooldown_rejection_count": _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT,
"threshold": bounded_cooldown_rejection_alert_threshold,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat()
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
else None,
},
)
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = True
logger.warning(
"scholar_search.cooldown_active",
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(runtime_state, now_utc)
if cooldown_remaining_seconds > 0:
runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
bounded_cooldown_rejection_alert_threshold = max(
1,
int(cooldown_rejection_alert_threshold),
)
if (
int(runtime_state.cooldown_rejection_count) >= bounded_cooldown_rejection_alert_threshold
and not bool(runtime_state.cooldown_alert_emitted)
):
logger.error(
"scholar_search.cooldown_rejection_threshold_exceeded",
extra={
"event": "scholar_search.cooldown_active",
"event": "scholar_search.cooldown_rejection_threshold_exceeded",
"query": normalized_query,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat()
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
"cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
"threshold": bounded_cooldown_rejection_alert_threshold,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat()
if runtime_state.cooldown_until
else None,
},
)
warning_codes = [
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
]
if _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED:
warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=warning_codes,
limit=bounded_limit,
)
runtime_state.cooldown_alert_emitted = True
runtime_state_updated = True
cached_entry = _cache_get_author_search_result(query_key, now_monotonic)
if cached_entry is not None:
cached = cached_entry.parsed
state_reason_override = (
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
)
logger.info(
"scholar_search.cache_hit",
extra={
"event": "scholar_search.cache_hit",
"query": normalized_query,
"state": cached.state.value,
"state_reason": cached.state_reason,
},
)
return _trim_author_search_result(
cached,
limit=bounded_limit,
extra_warnings=["author_search_served_from_cache"],
state_reason_override=state_reason_override,
)
logger.warning(
"scholar_search.cooldown_active",
extra={
"event": "scholar_search.cooldown_active",
"query": normalized_query,
"cooldown_remaining_seconds": cooldown_remaining_seconds,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat()
if runtime_state.cooldown_until
else None,
},
)
warning_codes = [
"author_search_cooldown_active",
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
]
if bool(runtime_state.cooldown_alert_emitted):
warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
if runtime_state_updated:
await db_session.commit()
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=warning_codes,
limit=bounded_limit,
)
cached = await _cache_get_author_search_result(
db_session,
query_key=query_key,
now_utc=now_utc,
)
if cached is not None:
state_reason_override = (
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
)
logger.info(
"scholar_search.cache_hit",
extra={
"event": "scholar_search.cache_hit",
"query": normalized_query,
"state": cached.state.value,
"state_reason": cached.state_reason,
},
)
await db_session.commit()
return _trim_author_search_result(
cached,
limit=bounded_limit,
extra_warnings=["author_search_served_from_cache"],
state_reason_override=state_reason_override,
)
if runtime_state.last_live_request_at is not None:
last_live_request_at = runtime_state.last_live_request_at
if last_live_request_at.tzinfo is None:
last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc)
runtime_state.last_live_request_at = last_live_request_at
runtime_state_updated = True
enforced_wait_seconds = (
(_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC + max(float(min_interval_seconds), 0.0))
- now_monotonic
last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
).total_seconds()
else:
enforced_wait_seconds = 0.0
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
sleep_seconds = max(0.0, float(enforced_wait_seconds)) + jitter_seconds
if sleep_seconds > 0.0:
logger.info(
"scholar_search.throttle_wait",
extra={
"event": "scholar_search.throttle_wait",
"query": normalized_query,
"sleep_seconds": round(sleep_seconds, 3),
},
)
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
sleep_seconds = max(0.0, enforced_wait_seconds) + jitter_seconds
if sleep_seconds > 0.0:
logger.info(
"scholar_search.throttle_wait",
extra={
"event": "scholar_search.throttle_wait",
"query": normalized_query,
"sleep_seconds": round(sleep_seconds, 3),
},
)
await asyncio.sleep(sleep_seconds)
await asyncio.sleep(sleep_seconds)
max_attempts = max(1, int(network_error_retries) + 1)
parsed: ParsedAuthorSearchPage | None = None
retry_warnings: list[str] = []
retry_scheduled_count = 0
max_attempts = max(1, int(network_error_retries) + 1)
parsed: ParsedAuthorSearchPage | None = None
retry_warnings: list[str] = []
retry_scheduled_count = 0
for attempt_index in range(max_attempts):
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
parsed = parse_author_search_page(fetch_result)
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
break
for attempt_index in range(max_attempts):
fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
parsed = parse_author_search_page(fetch_result)
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
break
retry_warnings.append("network_retry_scheduled_for_author_search")
retry_scheduled_count += 1
sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
if sleep_seconds > 0:
await asyncio.sleep(sleep_seconds)
retry_warnings.append("network_retry_scheduled_for_author_search")
retry_scheduled_count += 1
retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
if retry_sleep_seconds > 0:
await asyncio.sleep(retry_sleep_seconds)
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = time.monotonic()
runtime_state.last_live_request_at = datetime.now(timezone.utc)
runtime_state_updated = True
if parsed is None:
raise ScholarServiceError("Unable to complete scholar author search.")
if parsed is None:
raise ScholarServiceError("Unable to complete scholar author search.")
merged_parsed = replace(
parsed,
warnings=_merge_warnings(parsed.warnings, retry_warnings),
)
bounded_retry_alert_threshold = max(1, int(retry_alert_threshold))
if retry_scheduled_count >= bounded_retry_alert_threshold:
logger.warning(
"scholar_search.retry_threshold_exceeded",
extra={
"event": "scholar_search.retry_threshold_exceeded",
"query": normalized_query,
"retry_scheduled_count": retry_scheduled_count,
"threshold": bounded_retry_alert_threshold,
"final_state": merged_parsed.state.value,
"final_state_reason": merged_parsed.state_reason,
},
)
merged_parsed = replace(
parsed,
warnings=_merge_warnings(parsed.warnings, retry_warnings),
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
),
)
bounded_retry_alert_threshold = max(1, int(retry_alert_threshold))
if retry_scheduled_count >= bounded_retry_alert_threshold:
logger.warning(
"scholar_search.retry_threshold_exceeded",
extra={
"event": "scholar_search.retry_threshold_exceeded",
"query": normalized_query,
"retry_scheduled_count": retry_scheduled_count,
"threshold": bounded_retry_alert_threshold,
"final_state": merged_parsed.state.value,
"final_state_reason": merged_parsed.state_reason,
},
if _is_author_search_block_state(merged_parsed):
runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
logger.warning(
"scholar_search.block_detected",
extra={
"event": "scholar_search.block_detected",
"query": normalized_query,
"state_reason": merged_parsed.state_reason,
"consecutive_blocked_count": int(runtime_state.consecutive_blocked_count),
},
)
if int(runtime_state.consecutive_blocked_count) >= max(1, int(cooldown_block_threshold)):
runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(
seconds=max(60, int(cooldown_seconds))
)
runtime_state.consecutive_blocked_count = 0
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
["author_search_circuit_breaker_armed"],
),
)
if _is_author_search_block_state(merged_parsed):
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT += 1
logger.warning(
"scholar_search.block_detected",
logger.error(
"scholar_search.cooldown_activated",
extra={
"event": "scholar_search.block_detected",
"event": "scholar_search.cooldown_activated",
"query": normalized_query,
"state_reason": merged_parsed.state_reason,
"consecutive_blocked_count": _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT,
"cooldown_until_utc": runtime_state.cooldown_until.isoformat()
if runtime_state.cooldown_until
else None,
},
)
if _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT >= max(1, int(cooldown_block_threshold)):
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = datetime.now(timezone.utc) + timedelta(
seconds=max(60, int(cooldown_seconds))
)
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
["author_search_circuit_breaker_armed"],
),
)
logger.error(
"scholar_search.cooldown_activated",
extra={
"event": "scholar_search.cooldown_activated",
"query": normalized_query,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(),
},
)
else:
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
else:
runtime_state.consecutive_blocked_count = 0
ttl_seconds = (
min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
if _is_author_search_block_state(merged_parsed)
else max(1, int(cache_ttl_seconds))
)
_cache_set_author_search_result(
query_key=query_key,
parsed=merged_parsed,
ttl_seconds=float(ttl_seconds),
max_entries=cache_max_entries,
)
ttl_seconds = (
min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
if _is_author_search_block_state(merged_parsed)
else max(1, int(cache_ttl_seconds))
)
await _cache_set_author_search_result(
db_session,
query_key=query_key,
parsed=merged_parsed,
ttl_seconds=float(ttl_seconds),
max_entries=cache_max_entries,
now_utc=datetime.now(timezone.utc),
)
if runtime_state_updated:
await db_session.commit()
return _trim_author_search_result(
merged_parsed,

4
app/settings.py
View file

@ -64,6 +64,10 @@ class Settings:
"DATABASE_URL",
"postgresql+asyncpg://scholar:scholar@db:5432/scholar",
)
database_pool_mode: str = _env_str("DATABASE_POOL_MODE", "auto")
database_pool_size: int = _env_int("DATABASE_POOL_SIZE", 5)
database_pool_max_overflow: int = _env_int("DATABASE_POOL_MAX_OVERFLOW", 10)
database_pool_timeout_seconds: int = _env_int("DATABASE_POOL_TIMEOUT_SECONDS", 30)
session_secret_key: str = os.getenv("SESSION_SECRET_KEY", "dev-insecure-session-key")
session_cookie_secure: bool = _env_bool("SESSION_COOKIE_SECURE", False)
security_headers_enabled: bool = _env_bool("SECURITY_HEADERS_ENABLED", True)