remove legacy in-memory search state fallback #8

Merged
JustinZeus merged 1 commit from refactor/shared-author-search-state-phase2-cleanup into main 2026-02-19 22:31:45 +01:00
17 changed files with 766 additions and 248 deletions

1
.gitignore vendored
View file

@ -15,3 +15,4 @@ planning/
frontend/node_modules/ frontend/node_modules/
frontend/dist/ frontend/dist/
frontend/.vite/ frontend/.vite/
AGENTS.MD

View file

@ -31,7 +31,7 @@ This file is the consolidated source of truth for project scope, constraints, sc
- CSRF required for unsafe methods via `X-CSRF-Token`. - CSRF required for unsafe methods via `X-CSRF-Token`.
- Logging: - Logging:
- structured request logging with request IDs. - structured request logging with request IDs.
- redaction controls for sensitive fields. - redaction controls for sensitive fields.
## 4. Scraping Contract (Probe-Based) ## 4. Scraping Contract (Probe-Based)

View file

@ -0,0 +1,115 @@
"""Add shared author-search runtime and cache tables.
Revision ID: 20260219_0010
Revises: 20260219_0009
Create Date: 2026-02-19 22:20:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = "20260219_0010"
down_revision: str | Sequence[str] | None = "20260219_0009"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
table_names = set(inspector.get_table_names())
if "author_search_runtime_state" not in table_names:
op.create_table(
"author_search_runtime_state",
sa.Column("state_key", sa.String(length=64), nullable=False),
sa.Column("last_live_request_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("cooldown_until", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"consecutive_blocked_count",
sa.Integer(),
nullable=False,
server_default=sa.text("0"),
),
sa.Column(
"cooldown_rejection_count",
sa.Integer(),
nullable=False,
server_default=sa.text("0"),
),
sa.Column(
"cooldown_alert_emitted",
sa.Boolean(),
nullable=False,
server_default=sa.text("false"),
),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.PrimaryKeyConstraint("state_key", name=op.f("pk_author_search_runtime_state")),
)
if "author_search_cache_entries" not in table_names:
op.create_table(
"author_search_cache_entries",
sa.Column("query_key", sa.String(length=256), nullable=False),
sa.Column(
"payload",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column(
"cached_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.now(),
),
sa.PrimaryKeyConstraint("query_key", name=op.f("pk_author_search_cache_entries")),
)
op.create_index(
"ix_author_search_cache_expires_at",
"author_search_cache_entries",
["expires_at"],
unique=False,
)
op.create_index(
"ix_author_search_cache_cached_at",
"author_search_cache_entries",
["cached_at"],
unique=False,
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
table_names = set(inspector.get_table_names())
if "author_search_cache_entries" in table_names:
op.drop_index("ix_author_search_cache_cached_at", table_name="author_search_cache_entries")
op.drop_index("ix_author_search_cache_expires_at", table_name="author_search_cache_entries")
op.drop_table("author_search_cache_entries")
if "author_search_runtime_state" in table_names:
op.drop_table("author_search_runtime_state")

View file

@ -149,12 +149,14 @@ async def search_scholars(
request: Request, request: Request,
query: str = Query(..., min_length=2, max_length=120), query: str = Query(..., min_length=2, max_length=120),
limit: int = Query(10, ge=1, le=25), limit: int = Query(10, ge=1, le=25),
db_session: AsyncSession = Depends(get_db_session),
source: ScholarSource = Depends(get_scholar_source), source: ScholarSource = Depends(get_scholar_source),
current_user: User = Depends(get_api_current_user), current_user: User = Depends(get_api_current_user),
): ):
try: try:
parsed = await scholar_service.search_author_candidates( parsed = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query=query, query=query,
limit=limit, limit=limit,
network_error_retries=settings.ingestion_network_error_retries, network_error_retries=settings.ingestion_network_error_retries,

View file

@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from collections import defaultdict, deque from collections import deque
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from math import ceil from math import ceil
@ -25,14 +25,19 @@ class SlidingWindowRateLimiter:
self._max_attempts = max_attempts self._max_attempts = max_attempts
self._window_seconds = window_seconds self._window_seconds = window_seconds
self._now = now self._now = now
self._attempts: dict[str, deque[float]] = defaultdict(deque) self._attempts: dict[str, deque[float]] = {}
self._lock = Lock() self._lock = Lock()
def check(self, key: str) -> RateLimitDecision: def check(self, key: str) -> RateLimitDecision:
with self._lock: with self._lock:
now_value = self._now() now_value = self._now()
attempts = self._attempts[key] attempts = self._attempts.get(key)
if attempts is None:
return RateLimitDecision(allowed=True)
self._trim_expired(attempts, now_value) self._trim_expired(attempts, now_value)
if not attempts:
self._attempts.pop(key, None)
return RateLimitDecision(allowed=True)
if len(attempts) >= self._max_attempts: if len(attempts) >= self._max_attempts:
retry_after = self._window_seconds - (now_value - attempts[0]) retry_after = self._window_seconds - (now_value - attempts[0])
return RateLimitDecision( return RateLimitDecision(
@ -44,7 +49,10 @@ class SlidingWindowRateLimiter:
def record_failure(self, key: str) -> None: def record_failure(self, key: str) -> None:
with self._lock: with self._lock:
now_value = self._now() now_value = self._now()
attempts = self._attempts[key] attempts = self._attempts.get(key)
if attempts is None:
attempts = deque()
self._attempts[key] = attempts
self._trim_expired(attempts, now_value) self._trim_expired(attempts, now_value)
attempts.append(now_value) attempts.append(now_value)
@ -59,4 +67,3 @@ class SlidingWindowRateLimiter:
def _trim_expired(self, attempts: deque[float], now_value: float) -> None: def _trim_expired(self, attempts: deque[float], now_value: float) -> None:
while attempts and now_value - attempts[0] >= self._window_seconds: while attempts and now_value - attempts[0] >= self._window_seconds:
attempts.popleft() attempts.popleft()

View file

@ -319,3 +319,56 @@ class IngestionQueueItem(Base):
updated_at: Mapped[datetime] = mapped_column( updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now() DateTime(timezone=True), nullable=False, server_default=func.now()
) )
class AuthorSearchRuntimeState(Base):
__tablename__ = "author_search_runtime_state"
state_key: Mapped[str] = mapped_column(String(64), primary_key=True)
last_live_request_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
cooldown_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
consecutive_blocked_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
cooldown_rejection_count: Mapped[int] = mapped_column(
Integer,
nullable=False,
server_default=text("0"),
)
cooldown_alert_emitted: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
server_default=text("false"),
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
class AuthorSearchCacheEntry(Base):
__tablename__ = "author_search_cache_entries"
__table_args__ = (
Index("ix_author_search_cache_expires_at", "expires_at"),
Index("ix_author_search_cache_cached_at", "cached_at"),
)
query_key: Mapped[str] = mapped_column(String(256), primary_key=True)
payload: Mapped[dict] = mapped_column(
JSONB,
nullable=False,
)
expires_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
)
cached_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now()
)

View file

@ -1,5 +1,6 @@
from collections.abc import AsyncIterator from collections.abc import AsyncIterator
import logging import logging
import os
from sqlalchemy import text from sqlalchemy import text
from sqlalchemy.ext.asyncio import ( from sqlalchemy.ext.asyncio import (
@ -18,16 +19,50 @@ _engine: AsyncEngine | None = None
_session_factory: async_sessionmaker[AsyncSession] | None = None _session_factory: async_sessionmaker[AsyncSession] | None = None
def _normalized_pool_mode(raw_mode: str) -> str:
mode = (raw_mode or "").strip().lower()
if mode == "auto":
if os.getenv("PYTEST_CURRENT_TEST"):
return "null"
app_env = (os.getenv("APP_ENV") or "").strip().lower()
if app_env in {"test", "development", "dev", "local"}:
return "null"
return "queue"
if mode in {"null", "queue"}:
return mode
logger.warning(
"db.invalid_pool_mode_fallback",
extra={
"event": "db.invalid_pool_mode_fallback",
"database_pool_mode": raw_mode,
"fallback_mode": "queue",
},
)
return "queue"
def get_engine() -> AsyncEngine: def get_engine() -> AsyncEngine:
global _engine global _engine
if _engine is None: if _engine is None:
# NullPool avoids cross-event-loop connection reuse in tests and dev tools. pool_mode = _normalized_pool_mode(settings.database_pool_mode)
_engine = create_async_engine( engine_kwargs: dict[str, object] = {
settings.database_url, "pool_pre_ping": True,
pool_pre_ping=True, }
poolclass=NullPool, if pool_mode == "null":
engine_kwargs["poolclass"] = NullPool
else:
engine_kwargs["pool_size"] = max(1, int(settings.database_pool_size))
engine_kwargs["max_overflow"] = max(0, int(settings.database_pool_max_overflow))
engine_kwargs["pool_timeout"] = max(1, int(settings.database_pool_timeout_seconds))
_engine = create_async_engine(settings.database_url, **engine_kwargs)
logger.info(
"db.engine_initialized",
extra={
"event": "db.engine_initialized",
"pool_mode": pool_mode,
},
) )
logger.info("db.engine_initialized", extra={"event": "db.engine_initialized"})
return _engine return _engine

View file

@ -195,7 +195,7 @@ def _build_profile_url(*, scholar_id: str, cstart: int, pagesize: int) -> str:
query: dict[str, int | str] = {"hl": "en", "user": scholar_id} query: dict[str, int | str] = {"hl": "en", "user": scholar_id}
if cstart > 0: if cstart > 0:
query["cstart"] = int(cstart) query["cstart"] = int(cstart)
if pagesize > 0 and cstart > 0: if pagesize > 0:
query["pagesize"] = int(pagesize) query["pagesize"] = int(pagesize)
return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}" return f"{SCHOLAR_PROFILE_URL}?{urlencode(query)}"

View file

@ -1,8 +1,6 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
from collections import OrderedDict
from dataclasses import dataclass
from dataclasses import replace from dataclasses import replace
from datetime import datetime from datetime import datetime
from datetime import timedelta from datetime import timedelta
@ -11,16 +9,15 @@ import logging
import os import os
import random import random
import re import re
import time
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
from uuid import uuid4 from uuid import uuid4
from sqlalchemy import select from sqlalchemy import delete, func, select, text
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import ScholarProfile from app.db.models import AuthorSearchCacheEntry, AuthorSearchRuntimeState, ScholarProfile
from app.services.scholar_parser import ( from app.services.scholar_parser import (
ParseState, ParseState,
ParsedAuthorSearchPage, ParsedAuthorSearchPage,
@ -40,6 +37,9 @@ DEFAULT_AUTHOR_SEARCH_MIN_INTERVAL_SECONDS = 3.0
DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0 DEFAULT_AUTHOR_SEARCH_INTERVAL_JITTER_SECONDS = 1.0
DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2 DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD = 2
DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3 DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD = 3
AUTHOR_SEARCH_RUNTIME_STATE_KEY = "global"
AUTHOR_SEARCH_LOCK_NAMESPACE = 3901
AUTHOR_SEARCH_LOCK_KEY = 1
ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = { ALLOWED_IMAGE_UPLOAD_CONTENT_TYPES = {
"image/jpeg": ".jpg", "image/jpeg": ".jpg",
"image/png": ".png", "image/png": ".png",
@ -91,22 +91,6 @@ STATE_REASON_HINTS: dict[str, str] = {
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class _AuthorSearchCacheEntry:
parsed: ParsedAuthorSearchPage
expires_at_monotonic: float
cached_at_utc: datetime
_AUTHOR_SEARCH_EXECUTION_LOCK = asyncio.Lock()
_AUTHOR_SEARCH_CACHE: OrderedDict[str, _AuthorSearchCacheEntry] = OrderedDict()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC: datetime | None = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
class ScholarServiceError(ValueError): class ScholarServiceError(ValueError):
"""Raised for expected scholar-management validation failures.""" """Raised for expected scholar-management validation failures."""
@ -235,64 +219,260 @@ def _policy_blocked_author_search_result(
) )
def _cache_get_author_search_result(query_key: str, now_monotonic: float) -> _AuthorSearchCacheEntry | None: async def _acquire_author_search_lock(db_session: AsyncSession) -> None:
entry = _AUTHOR_SEARCH_CACHE.get(query_key) await db_session.execute(
text("SELECT pg_advisory_xact_lock(:namespace, :lock_key)"),
{
"namespace": AUTHOR_SEARCH_LOCK_NAMESPACE,
"lock_key": AUTHOR_SEARCH_LOCK_KEY,
},
)
async def _load_runtime_state_for_update(
db_session: AsyncSession,
) -> AuthorSearchRuntimeState:
result = await db_session.execute(
select(AuthorSearchRuntimeState)
.where(AuthorSearchRuntimeState.state_key == AUTHOR_SEARCH_RUNTIME_STATE_KEY)
.with_for_update()
)
state = result.scalar_one_or_none()
if state is not None:
return state
state = AuthorSearchRuntimeState(state_key=AUTHOR_SEARCH_RUNTIME_STATE_KEY)
db_session.add(state)
await db_session.flush()
return state
def _serialize_parsed_author_search_page(parsed: ParsedAuthorSearchPage) -> dict:
return {
"state": parsed.state.value,
"state_reason": parsed.state_reason,
"marker_counts": {str(key): int(value) for key, value in parsed.marker_counts.items()},
"warnings": [str(value) for value in parsed.warnings],
"candidates": [
{
"scholar_id": candidate.scholar_id,
"display_name": candidate.display_name,
"affiliation": candidate.affiliation,
"email_domain": candidate.email_domain,
"cited_by_count": candidate.cited_by_count,
"interests": [str(interest) for interest in candidate.interests],
"profile_url": candidate.profile_url,
"profile_image_url": candidate.profile_image_url,
}
for candidate in parsed.candidates
],
}
def _deserialize_parsed_author_search_page(payload: object) -> ParsedAuthorSearchPage | None:
if not isinstance(payload, dict):
return None
state_raw = str(payload.get("state", "")).strip()
try:
state = ParseState(state_raw)
except ValueError:
return None
marker_counts_payload = payload.get("marker_counts")
marker_counts = (
{str(key): int(value) for key, value in marker_counts_payload.items()}
if isinstance(marker_counts_payload, dict)
else {}
)
warnings_payload = payload.get("warnings")
warnings = (
[str(value) for value in warnings_payload if isinstance(value, str)]
if isinstance(warnings_payload, list)
else []
)
from app.services.scholar_parser import ScholarSearchCandidate
candidates_payload = payload.get("candidates")
normalized_candidates = []
for value in candidates_payload if isinstance(candidates_payload, list) else []:
if not isinstance(value, dict):
continue
scholar_id = str(value.get("scholar_id", "")).strip()
display_name = str(value.get("display_name", "")).strip()
profile_url = str(value.get("profile_url", "")).strip()
if not scholar_id or not display_name or not profile_url:
continue
interests_payload = value.get("interests")
interests = (
[str(item) for item in interests_payload if isinstance(item, str)]
if isinstance(interests_payload, list)
else []
)
cited_by_count = value.get("cited_by_count")
parsed_cited_by_count: int | None = None
if isinstance(cited_by_count, int):
parsed_cited_by_count = cited_by_count
elif isinstance(cited_by_count, str) and cited_by_count.strip():
try:
parsed_cited_by_count = int(cited_by_count)
except ValueError:
parsed_cited_by_count = None
normalized_candidates.append(
{
"scholar_id": scholar_id,
"display_name": display_name,
"affiliation": str(value.get("affiliation")).strip()
if value.get("affiliation") is not None
else None,
"email_domain": str(value.get("email_domain")).strip()
if value.get("email_domain") is not None
else None,
"cited_by_count": parsed_cited_by_count,
"interests": interests,
"profile_url": profile_url,
"profile_image_url": str(value.get("profile_image_url")).strip()
if value.get("profile_image_url") is not None
else None,
}
)
return ParsedAuthorSearchPage(
state=state,
state_reason=str(payload.get("state_reason", "")).strip() or "unknown",
candidates=[
ScholarSearchCandidate(
scholar_id=item["scholar_id"],
display_name=item["display_name"],
affiliation=item["affiliation"],
email_domain=item["email_domain"],
cited_by_count=item["cited_by_count"],
interests=item["interests"],
profile_url=item["profile_url"],
profile_image_url=item["profile_image_url"],
)
for item in normalized_candidates
],
marker_counts=marker_counts,
warnings=warnings,
)
async def _cache_get_author_search_result(
db_session: AsyncSession,
*,
query_key: str,
now_utc: datetime,
) -> ParsedAuthorSearchPage | None:
result = await db_session.execute(
select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
)
entry = result.scalar_one_or_none()
if entry is None: if entry is None:
return None return None
if entry.expires_at_monotonic <= now_monotonic: expires_at = entry.expires_at
_AUTHOR_SEARCH_CACHE.pop(query_key, None) if expires_at.tzinfo is None:
expires_at = expires_at.replace(tzinfo=timezone.utc)
if expires_at <= now_utc:
await db_session.delete(entry)
return None return None
_AUTHOR_SEARCH_CACHE.move_to_end(query_key) parsed = _deserialize_parsed_author_search_page(entry.payload)
return entry if parsed is None:
await db_session.delete(entry)
return None
return parsed
def _cache_set_author_search_result( async def _cache_set_author_search_result(
db_session: AsyncSession,
*, *,
query_key: str, query_key: str,
parsed: ParsedAuthorSearchPage, parsed: ParsedAuthorSearchPage,
ttl_seconds: float, ttl_seconds: float,
max_entries: int, max_entries: int,
now_utc: datetime,
) -> None: ) -> None:
ttl = max(float(ttl_seconds), 0.0) ttl = max(float(ttl_seconds), 0.0)
existing_result = await db_session.execute(
select(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key == query_key)
)
existing = existing_result.scalar_one_or_none()
if ttl <= 0.0: if ttl <= 0.0:
_AUTHOR_SEARCH_CACHE.pop(query_key, None) if existing is not None:
await db_session.delete(existing)
return return
_AUTHOR_SEARCH_CACHE[query_key] = _AuthorSearchCacheEntry( expires_at = now_utc + timedelta(seconds=ttl)
parsed=parsed, payload = _serialize_parsed_author_search_page(parsed)
expires_at_monotonic=time.monotonic() + ttl, if existing is None:
cached_at_utc=datetime.now(timezone.utc), db_session.add(
) AuthorSearchCacheEntry(
_AUTHOR_SEARCH_CACHE.move_to_end(query_key) query_key=query_key,
payload=payload,
expires_at=expires_at,
cached_at=now_utc,
updated_at=now_utc,
)
)
else:
existing.payload = payload
existing.expires_at = expires_at
existing.cached_at = now_utc
existing.updated_at = now_utc
await _prune_author_search_cache(db_session, now_utc=now_utc, max_entries=max_entries)
async def _prune_author_search_cache(
db_session: AsyncSession,
*,
now_utc: datetime,
max_entries: int,
) -> None:
await db_session.execute(
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.expires_at <= now_utc)
)
bounded_max_entries = max(1, int(max_entries)) bounded_max_entries = max(1, int(max_entries))
while len(_AUTHOR_SEARCH_CACHE) > bounded_max_entries: count_result = await db_session.execute(
_AUTHOR_SEARCH_CACHE.popitem(last=False) select(func.count()).select_from(AuthorSearchCacheEntry)
)
entry_count = int(count_result.scalar_one() or 0)
overflow = max(0, entry_count - bounded_max_entries)
if overflow <= 0:
return
stale_keys_result = await db_session.execute(
select(AuthorSearchCacheEntry.query_key)
.order_by(AuthorSearchCacheEntry.cached_at.asc())
.limit(overflow)
)
stale_keys = [str(row[0]) for row in stale_keys_result.all()]
if stale_keys:
await db_session.execute(
delete(AuthorSearchCacheEntry).where(AuthorSearchCacheEntry.query_key.in_(stale_keys))
)
def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool: def _is_author_search_block_state(parsed: ParsedAuthorSearchPage) -> bool:
return parsed.state == ParseState.BLOCKED_OR_CAPTCHA return parsed.state == ParseState.BLOCKED_OR_CAPTCHA
def _author_search_cooldown_remaining_seconds(now_utc: datetime) -> int: def _author_search_cooldown_remaining_seconds(
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is None: runtime_state: AuthorSearchRuntimeState,
now_utc: datetime,
) -> int:
cooldown_until = runtime_state.cooldown_until
if cooldown_until is None:
return 0 return 0
remaining_seconds = int((_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC - now_utc).total_seconds()) if cooldown_until.tzinfo is None:
cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
remaining_seconds = int((cooldown_until - now_utc).total_seconds())
return max(0, remaining_seconds) return max(0, remaining_seconds)
def _reset_author_search_runtime_state_for_tests() -> None: def _reset_author_search_runtime_state_for_tests() -> None:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC # Runtime state now lives in the database; tests should reset via DB fixtures.
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC return None
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
global _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
global _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
_AUTHOR_SEARCH_CACHE.clear()
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = 0.0
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
async def list_scholars_for_user( async def list_scholars_for_user(
@ -375,6 +555,7 @@ async def delete_scholar(
async def search_author_candidates( async def search_author_candidates(
*, *,
source: ScholarSource, source: ScholarSource,
db_session: AsyncSession,
query: str, query: str,
limit: int, limit: int,
network_error_retries: int = 1, network_error_retries: int = 1,
@ -390,12 +571,6 @@ async def search_author_candidates(
retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD, retry_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_RETRY_ALERT_THRESHOLD,
cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD, cooldown_rejection_alert_threshold: int = DEFAULT_AUTHOR_SEARCH_COOLDOWN_REJECTION_ALERT_THRESHOLD,
) -> ParsedAuthorSearchPage: ) -> ParsedAuthorSearchPage:
global _AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC
global _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
global _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT
global _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT
global _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED
normalized_query = query.strip() normalized_query = query.strip()
if len(normalized_query) < 2: if len(normalized_query) < 2:
raise ScholarServiceError("Search query must be at least 2 characters.") raise ScholarServiceError("Search query must be at least 2 characters.")
@ -416,208 +591,234 @@ async def search_author_candidates(
limit=bounded_limit, limit=bounded_limit,
) )
async with _AUTHOR_SEARCH_EXECUTION_LOCK: await _acquire_author_search_lock(db_session)
now_utc = datetime.now(timezone.utc) runtime_state = await _load_runtime_state_for_update(db_session)
now_monotonic = time.monotonic() runtime_state_updated = False
now_utc = datetime.now(timezone.utc)
if ( if runtime_state.cooldown_until is not None:
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC is not None cooldown_until = runtime_state.cooldown_until
and now_utc >= _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC if cooldown_until.tzinfo is None:
): cooldown_until = cooldown_until.replace(tzinfo=timezone.utc)
runtime_state.cooldown_until = cooldown_until
runtime_state_updated = True
if now_utc >= cooldown_until:
logger.info( logger.info(
"scholar_search.cooldown_expired", "scholar_search.cooldown_expired",
extra={ extra={
"event": "scholar_search.cooldown_expired", "event": "scholar_search.cooldown_expired",
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(), "cooldown_until_utc": cooldown_until.isoformat(),
}, },
) )
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = None runtime_state.cooldown_until = None
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0 runtime_state.cooldown_rejection_count = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False runtime_state.cooldown_alert_emitted = False
runtime_state_updated = True
cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(now_utc) cooldown_remaining_seconds = _author_search_cooldown_remaining_seconds(runtime_state, now_utc)
if cooldown_remaining_seconds > 0: if cooldown_remaining_seconds > 0:
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT += 1 runtime_state.cooldown_rejection_count = int(runtime_state.cooldown_rejection_count) + 1
bounded_cooldown_rejection_alert_threshold = max( bounded_cooldown_rejection_alert_threshold = max(
1, 1,
int(cooldown_rejection_alert_threshold), int(cooldown_rejection_alert_threshold),
) )
if ( if (
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT int(runtime_state.cooldown_rejection_count) >= bounded_cooldown_rejection_alert_threshold
>= bounded_cooldown_rejection_alert_threshold and not bool(runtime_state.cooldown_alert_emitted)
and not _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED ):
): logger.error(
logger.error( "scholar_search.cooldown_rejection_threshold_exceeded",
"scholar_search.cooldown_rejection_threshold_exceeded",
extra={
"event": "scholar_search.cooldown_rejection_threshold_exceeded",
"query": normalized_query,
"cooldown_rejection_count": _AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT,
"threshold": bounded_cooldown_rejection_alert_threshold,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat()
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC
else None,
},
)
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = True
logger.warning(
"scholar_search.cooldown_active",
extra={ extra={
"event": "scholar_search.cooldown_active", "event": "scholar_search.cooldown_rejection_threshold_exceeded",
"query": normalized_query, "query": normalized_query,
"cooldown_remaining_seconds": cooldown_remaining_seconds, "cooldown_rejection_count": int(runtime_state.cooldown_rejection_count),
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat() "threshold": bounded_cooldown_rejection_alert_threshold,
if _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC "cooldown_until_utc": runtime_state.cooldown_until.isoformat()
if runtime_state.cooldown_until
else None, else None,
}, },
) )
warning_codes = [ runtime_state.cooldown_alert_emitted = True
"author_search_cooldown_active", runtime_state_updated = True
f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
]
if _AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED:
warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
return _policy_blocked_author_search_result(
reason=SEARCH_COOLDOWN_REASON,
warning_codes=warning_codes,
limit=bounded_limit,
)
cached_entry = _cache_get_author_search_result(query_key, now_monotonic) logger.warning(
if cached_entry is not None: "scholar_search.cooldown_active",
cached = cached_entry.parsed extra={
state_reason_override = ( "event": "scholar_search.cooldown_active",
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None "query": normalized_query,
) "cooldown_remaining_seconds": cooldown_remaining_seconds,
logger.info( "cooldown_until_utc": runtime_state.cooldown_until.isoformat()
"scholar_search.cache_hit", if runtime_state.cooldown_until
extra={ else None,
"event": "scholar_search.cache_hit", },
"query": normalized_query, )
"state": cached.state.value, warning_codes = [
"state_reason": cached.state_reason, "author_search_cooldown_active",
}, f"author_search_cooldown_remaining_{cooldown_remaining_seconds}s",
) ]
return _trim_author_search_result( if bool(runtime_state.cooldown_alert_emitted):
cached, warning_codes.append("author_search_cooldown_alert_threshold_exceeded")
limit=bounded_limit, if runtime_state_updated:
extra_warnings=["author_search_served_from_cache"], await db_session.commit()
state_reason_override=state_reason_override, return _policy_blocked_author_search_result(
) reason=SEARCH_COOLDOWN_REASON,
warning_codes=warning_codes,
limit=bounded_limit,
)
cached = await _cache_get_author_search_result(
db_session,
query_key=query_key,
now_utc=now_utc,
)
if cached is not None:
state_reason_override = (
SEARCH_CACHED_BLOCK_REASON if _is_author_search_block_state(cached) else None
)
logger.info(
"scholar_search.cache_hit",
extra={
"event": "scholar_search.cache_hit",
"query": normalized_query,
"state": cached.state.value,
"state_reason": cached.state_reason,
},
)
await db_session.commit()
return _trim_author_search_result(
cached,
limit=bounded_limit,
extra_warnings=["author_search_served_from_cache"],
state_reason_override=state_reason_override,
)
if runtime_state.last_live_request_at is not None:
last_live_request_at = runtime_state.last_live_request_at
if last_live_request_at.tzinfo is None:
last_live_request_at = last_live_request_at.replace(tzinfo=timezone.utc)
runtime_state.last_live_request_at = last_live_request_at
runtime_state_updated = True
enforced_wait_seconds = ( enforced_wait_seconds = (
(_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC + max(float(min_interval_seconds), 0.0)) last_live_request_at + timedelta(seconds=max(float(min_interval_seconds), 0.0)) - now_utc
- now_monotonic ).total_seconds()
else:
enforced_wait_seconds = 0.0
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0))
sleep_seconds = max(0.0, float(enforced_wait_seconds)) + jitter_seconds
if sleep_seconds > 0.0:
logger.info(
"scholar_search.throttle_wait",
extra={
"event": "scholar_search.throttle_wait",
"query": normalized_query,
"sleep_seconds": round(sleep_seconds, 3),
},
) )
jitter_seconds = random.uniform(0.0, max(float(interval_jitter_seconds), 0.0)) await asyncio.sleep(sleep_seconds)
sleep_seconds = max(0.0, enforced_wait_seconds) + jitter_seconds
if sleep_seconds > 0.0:
logger.info(
"scholar_search.throttle_wait",
extra={
"event": "scholar_search.throttle_wait",
"query": normalized_query,
"sleep_seconds": round(sleep_seconds, 3),
},
)
await asyncio.sleep(sleep_seconds)
max_attempts = max(1, int(network_error_retries) + 1) max_attempts = max(1, int(network_error_retries) + 1)
parsed: ParsedAuthorSearchPage | None = None parsed: ParsedAuthorSearchPage | None = None
retry_warnings: list[str] = [] retry_warnings: list[str] = []
retry_scheduled_count = 0 retry_scheduled_count = 0
for attempt_index in range(max_attempts): for attempt_index in range(max_attempts):
fetch_result = await source.fetch_author_search_html(normalized_query, start=0) fetch_result = await source.fetch_author_search_html(normalized_query, start=0)
parsed = parse_author_search_page(fetch_result) parsed = parse_author_search_page(fetch_result)
if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1: if parsed.state != ParseState.NETWORK_ERROR or attempt_index >= max_attempts - 1:
break break
retry_warnings.append("network_retry_scheduled_for_author_search") retry_warnings.append("network_retry_scheduled_for_author_search")
retry_scheduled_count += 1 retry_scheduled_count += 1
sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index) retry_sleep_seconds = max(float(retry_backoff_seconds), 0.0) * (2**attempt_index)
if sleep_seconds > 0: if retry_sleep_seconds > 0:
await asyncio.sleep(sleep_seconds) await asyncio.sleep(retry_sleep_seconds)
_AUTHOR_SEARCH_LAST_LIVE_REQUEST_MONOTONIC = time.monotonic() runtime_state.last_live_request_at = datetime.now(timezone.utc)
runtime_state_updated = True
if parsed is None: if parsed is None:
raise ScholarServiceError("Unable to complete scholar author search.") raise ScholarServiceError("Unable to complete scholar author search.")
merged_parsed = replace(
parsed,
warnings=_merge_warnings(parsed.warnings, retry_warnings),
)
bounded_retry_alert_threshold = max(1, int(retry_alert_threshold))
if retry_scheduled_count >= bounded_retry_alert_threshold:
logger.warning(
"scholar_search.retry_threshold_exceeded",
extra={
"event": "scholar_search.retry_threshold_exceeded",
"query": normalized_query,
"retry_scheduled_count": retry_scheduled_count,
"threshold": bounded_retry_alert_threshold,
"final_state": merged_parsed.state.value,
"final_state_reason": merged_parsed.state_reason,
},
)
merged_parsed = replace( merged_parsed = replace(
parsed, merged_parsed,
warnings=_merge_warnings(parsed.warnings, retry_warnings), warnings=_merge_warnings(
merged_parsed.warnings,
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"],
),
) )
bounded_retry_alert_threshold = max(1, int(retry_alert_threshold)) if _is_author_search_block_state(merged_parsed):
if retry_scheduled_count >= bounded_retry_alert_threshold: runtime_state.consecutive_blocked_count = int(runtime_state.consecutive_blocked_count) + 1
logger.warning( logger.warning(
"scholar_search.retry_threshold_exceeded", "scholar_search.block_detected",
extra={ extra={
"event": "scholar_search.retry_threshold_exceeded", "event": "scholar_search.block_detected",
"query": normalized_query, "query": normalized_query,
"retry_scheduled_count": retry_scheduled_count, "state_reason": merged_parsed.state_reason,
"threshold": bounded_retry_alert_threshold, "consecutive_blocked_count": int(runtime_state.consecutive_blocked_count),
"final_state": merged_parsed.state.value, },
"final_state_reason": merged_parsed.state_reason, )
}, if int(runtime_state.consecutive_blocked_count) >= max(1, int(cooldown_block_threshold)):
runtime_state.cooldown_until = datetime.now(timezone.utc) + timedelta(
seconds=max(60, int(cooldown_seconds))
) )
runtime_state.consecutive_blocked_count = 0
runtime_state.cooldown_rejection_count = 0
runtime_state.cooldown_alert_emitted = False
merged_parsed = replace( merged_parsed = replace(
merged_parsed, merged_parsed,
warnings=_merge_warnings( warnings=_merge_warnings(
merged_parsed.warnings, merged_parsed.warnings,
[f"author_search_retry_threshold_exceeded_{retry_scheduled_count}"], ["author_search_circuit_breaker_armed"],
), ),
) )
logger.error(
if _is_author_search_block_state(merged_parsed): "scholar_search.cooldown_activated",
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT += 1
logger.warning(
"scholar_search.block_detected",
extra={ extra={
"event": "scholar_search.block_detected", "event": "scholar_search.cooldown_activated",
"query": normalized_query, "query": normalized_query,
"state_reason": merged_parsed.state_reason, "cooldown_until_utc": runtime_state.cooldown_until.isoformat()
"consecutive_blocked_count": _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT, if runtime_state.cooldown_until
else None,
}, },
) )
if _AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT >= max(1, int(cooldown_block_threshold)): else:
_AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC = datetime.now(timezone.utc) + timedelta( runtime_state.consecutive_blocked_count = 0
seconds=max(60, int(cooldown_seconds))
)
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_REJECTION_COUNT = 0
_AUTHOR_SEARCH_COOLDOWN_ALERT_EMITTED = False
merged_parsed = replace(
merged_parsed,
warnings=_merge_warnings(
merged_parsed.warnings,
["author_search_circuit_breaker_armed"],
),
)
logger.error(
"scholar_search.cooldown_activated",
extra={
"event": "scholar_search.cooldown_activated",
"query": normalized_query,
"cooldown_until_utc": _AUTHOR_SEARCH_COOLDOWN_UNTIL_UTC.isoformat(),
},
)
else:
_AUTHOR_SEARCH_CONSECUTIVE_BLOCKED_COUNT = 0
ttl_seconds = ( ttl_seconds = (
min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds))) min(max(1, int(blocked_cache_ttl_seconds)), max(1, int(cache_ttl_seconds)))
if _is_author_search_block_state(merged_parsed) if _is_author_search_block_state(merged_parsed)
else max(1, int(cache_ttl_seconds)) else max(1, int(cache_ttl_seconds))
) )
_cache_set_author_search_result( await _cache_set_author_search_result(
query_key=query_key, db_session,
parsed=merged_parsed, query_key=query_key,
ttl_seconds=float(ttl_seconds), parsed=merged_parsed,
max_entries=cache_max_entries, ttl_seconds=float(ttl_seconds),
) max_entries=cache_max_entries,
now_utc=datetime.now(timezone.utc),
)
if runtime_state_updated:
await db_session.commit()
return _trim_author_search_result( return _trim_author_search_result(
merged_parsed, merged_parsed,

View file

@ -64,6 +64,10 @@ class Settings:
"DATABASE_URL", "DATABASE_URL",
"postgresql+asyncpg://scholar:scholar@db:5432/scholar", "postgresql+asyncpg://scholar:scholar@db:5432/scholar",
) )
database_pool_mode: str = _env_str("DATABASE_POOL_MODE", "auto")
database_pool_size: int = _env_int("DATABASE_POOL_SIZE", 5)
database_pool_max_overflow: int = _env_int("DATABASE_POOL_MAX_OVERFLOW", 10)
database_pool_timeout_seconds: int = _env_int("DATABASE_POOL_TIMEOUT_SECONDS", 30)
session_secret_key: str = os.getenv("SESSION_SECRET_KEY", "dev-insecure-session-key") session_secret_key: str = os.getenv("SESSION_SECRET_KEY", "dev-insecure-session-key")
session_cookie_secure: bool = _env_bool("SESSION_COOKIE_SECURE", False) session_cookie_secure: bool = _env_bool("SESSION_COOKIE_SECURE", False)
security_headers_enabled: bool = _env_bool("SECURITY_HEADERS_ENABLED", True) security_headers_enabled: bool = _env_bool("SECURITY_HEADERS_ENABLED", True)

View file

@ -0,0 +1,50 @@
import { beforeEach, describe, expect, it } from "vitest";
import { createPinia, setActivePinia } from "pinia";
import { useUserSettingsStore } from "@/stores/user_settings";
describe("user settings store", () => {
beforeEach(() => {
setActivePinia(createPinia());
});
it("falls back network failure threshold to 2 when policy value is missing", () => {
const store = useUserSettingsStore();
store.applySettings({
auto_run_enabled: true,
run_interval_minutes: 60,
request_delay_seconds: 2,
nav_visible_pages: ["dashboard", "scholars", "settings"],
policy: {
min_run_interval_minutes: 15,
min_request_delay_seconds: 2,
automation_allowed: true,
manual_run_allowed: true,
blocked_failure_threshold: 1,
network_failure_threshold: Number.NaN,
cooldown_blocked_seconds: 1800,
cooldown_network_seconds: 900,
},
safety_state: {
cooldown_active: false,
cooldown_reason: null,
cooldown_reason_label: null,
cooldown_until: null,
cooldown_remaining_seconds: 0,
recommended_action: null,
counters: {
consecutive_blocked_runs: 0,
consecutive_network_runs: 0,
cooldown_entry_count: 0,
blocked_start_count: 0,
last_blocked_failure_count: 0,
last_network_failure_count: 0,
last_evaluated_run_id: null,
},
},
});
expect(store.networkFailureThreshold).toBe(2);
});
});

View file

@ -84,7 +84,7 @@ export const useUserSettingsStore = defineStore("userSettings", {
: 1; : 1;
this.networkFailureThreshold = Number.isFinite(settings.policy?.network_failure_threshold) this.networkFailureThreshold = Number.isFinite(settings.policy?.network_failure_threshold)
? Math.max(1, settings.policy.network_failure_threshold) ? Math.max(1, settings.policy.network_failure_threshold)
: 1; : 2;
this.cooldownBlockedSeconds = Number.isFinite(settings.policy?.cooldown_blocked_seconds) this.cooldownBlockedSeconds = Number.isFinite(settings.policy?.cooldown_blocked_seconds)
? Math.max(60, settings.policy.cooldown_blocked_seconds) ? Math.max(60, settings.policy.cooldown_blocked_seconds)
: 1800; : 1800;

View file

@ -19,6 +19,8 @@ from app.settings import settings
RESET_SQL = text( RESET_SQL = text(
""" """
TRUNCATE TABLE TRUNCATE TABLE
author_search_cache_entries,
author_search_runtime_state,
ingestion_queue_items, ingestion_queue_items,
scholar_publications, scholar_publications,
crawl_runs, crawl_runs,

View file

@ -11,10 +11,12 @@ EXPECTED_TABLES = {
"scholar_publications", "scholar_publications",
"crawl_runs", "crawl_runs",
"ingestion_queue_items", "ingestion_queue_items",
"author_search_runtime_state",
"author_search_cache_entries",
} }
EXPECTED_ENUMS = {"run_status", "run_trigger_type"} EXPECTED_ENUMS = {"run_status", "run_trigger_type"}
EXPECTED_REVISION = "20260219_0009" EXPECTED_REVISION = "20260219_0010"
@pytest.mark.integration @pytest.mark.integration

View file

@ -32,3 +32,22 @@ def test_rate_limiter_enforces_windowed_attempt_budget() -> None:
clock["now"] = 11.0 clock["now"] = 11.0
assert limiter.check(key).allowed assert limiter.check(key).allowed
def test_rate_limiter_check_does_not_create_or_retain_empty_keys() -> None:
clock = {"now": 0.0}
limiter = SlidingWindowRateLimiter(
max_attempts=2,
window_seconds=10,
now=lambda: clock["now"],
)
key = "127.0.0.1:never-failed@example.com"
assert limiter.check(key).allowed
assert key not in limiter._attempts
limiter.record_failure(key)
assert key in limiter._attempts
clock["now"] = 11.0
assert limiter.check(key).allowed
assert key not in limiter._attempts

View file

@ -1,11 +1,14 @@
from __future__ import annotations from __future__ import annotations
import pytest import pytest
from sqlalchemy.ext.asyncio import AsyncSession
from app.services import scholars as scholar_service from app.services import scholars as scholar_service
from app.services.scholar_parser import ParseState from app.services.scholar_parser import ParseState
from app.services.scholar_source import FetchResult from app.services.scholar_source import FetchResult
pytestmark = [pytest.mark.integration, pytest.mark.db]
class StubScholarSource: class StubScholarSource:
def __init__(self, fetch_results: list[FetchResult]) -> None: def __init__(self, fetch_results: list[FetchResult]) -> None:
@ -21,13 +24,6 @@ class StubScholarSource:
return self._fetch_results[index] return self._fetch_results[index]
@pytest.fixture(autouse=True)
def reset_author_search_runtime_state() -> None:
scholar_service._reset_author_search_runtime_state_for_tests()
yield
scholar_service._reset_author_search_runtime_state_for_tests()
def _ok_author_search_fetch() -> FetchResult: def _ok_author_search_fetch() -> FetchResult:
body = ( body = (
"<html><body>" "<html><body>"
@ -74,11 +70,14 @@ def _network_timeout_fetch() -> FetchResult:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_search_author_candidates_serves_cached_response_for_same_query() -> None: async def test_search_author_candidates_serves_cached_response_for_same_query(
db_session: AsyncSession,
) -> None:
source = StubScholarSource([_ok_author_search_fetch()]) source = StubScholarSource([_ok_author_search_fetch()])
first = await scholar_service.search_author_candidates( first = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Ada Lovelace", query="Ada Lovelace",
limit=10, limit=10,
network_error_retries=0, network_error_retries=0,
@ -94,6 +93,7 @@ async def test_search_author_candidates_serves_cached_response_for_same_query()
) )
second = await scholar_service.search_author_candidates( second = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Ada Lovelace", query="Ada Lovelace",
limit=10, limit=10,
network_error_retries=0, network_error_retries=0,
@ -117,11 +117,14 @@ async def test_search_author_candidates_serves_cached_response_for_same_query()
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_search_author_candidates_trips_cooldown_after_blocked_responses() -> None: async def test_search_author_candidates_trips_cooldown_after_blocked_responses(
db_session: AsyncSession,
) -> None:
source = StubScholarSource([_blocked_author_search_fetch()]) source = StubScholarSource([_blocked_author_search_fetch()])
first = await scholar_service.search_author_candidates( first = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Blocked Query", query="Blocked Query",
limit=10, limit=10,
network_error_retries=0, network_error_retries=0,
@ -137,6 +140,7 @@ async def test_search_author_candidates_trips_cooldown_after_blocked_responses()
) )
second = await scholar_service.search_author_candidates( second = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Blocked Query", query="Blocked Query",
limit=10, limit=10,
network_error_retries=0, network_error_retries=0,
@ -161,11 +165,14 @@ async def test_search_author_candidates_trips_cooldown_after_blocked_responses()
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_search_author_candidates_emits_cooldown_alert_warning_after_threshold() -> None: async def test_search_author_candidates_emits_cooldown_alert_warning_after_threshold(
db_session: AsyncSession,
) -> None:
source = StubScholarSource([_blocked_author_search_fetch()]) source = StubScholarSource([_blocked_author_search_fetch()])
_ = await scholar_service.search_author_candidates( _ = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Blocked Query", query="Blocked Query",
limit=10, limit=10,
network_error_retries=0, network_error_retries=0,
@ -182,6 +189,7 @@ async def test_search_author_candidates_emits_cooldown_alert_warning_after_thres
) )
second = await scholar_service.search_author_candidates( second = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Blocked Query", query="Blocked Query",
limit=10, limit=10,
network_error_retries=0, network_error_retries=0,
@ -202,7 +210,9 @@ async def test_search_author_candidates_emits_cooldown_alert_warning_after_thres
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_search_author_candidates_adds_retry_threshold_warning() -> None: async def test_search_author_candidates_adds_retry_threshold_warning(
db_session: AsyncSession,
) -> None:
source = StubScholarSource( source = StubScholarSource(
[ [
_network_timeout_fetch(), _network_timeout_fetch(),
@ -213,6 +223,7 @@ async def test_search_author_candidates_adds_retry_threshold_warning() -> None:
result = await scholar_service.search_author_candidates( result = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Ada Lovelace", query="Ada Lovelace",
limit=10, limit=10,
network_error_retries=2, network_error_retries=2,
@ -234,11 +245,14 @@ async def test_search_author_candidates_adds_retry_threshold_warning() -> None:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_search_author_candidates_can_be_disabled_by_configuration() -> None: async def test_search_author_candidates_can_be_disabled_by_configuration(
db_session: AsyncSession,
) -> None:
source = StubScholarSource([_ok_author_search_fetch()]) source = StubScholarSource([_ok_author_search_fetch()])
parsed = await scholar_service.search_author_candidates( parsed = await scholar_service.search_author_candidates(
source=source, source=source,
db_session=db_session,
query="Ada Lovelace", query="Ada Lovelace",
limit=5, limit=5,
network_error_retries=0, network_error_retries=0,

View file

@ -0,0 +1,13 @@
from app.services.scholar_source import _build_profile_url
def test_build_profile_url_includes_pagesize_for_initial_page() -> None:
url = _build_profile_url(
scholar_id="abcDEF123456",
cstart=0,
pagesize=100,
)
assert "user=abcDEF123456" in url
assert "pagesize=100" in url
assert "cstart=" not in url