justinzeus/scholarr
1
0
Fork 0
Code Issues 1 Pull requests 18 Projects Releases Packages Wiki Activity Actions

ci: add ruff linting and mypy type checking

Browse source 
Add ruff and mypy to dev dependencies with configuration in pyproject.toml.
Add a lint CI job that runs ruff check, ruff format --check, and mypy.
Auto-fix import sorting and formatting across the codebase. Exclude
alembic/versions from linting (auto-generated migrations). Ignore B008
(FastAPI Depends pattern) and RUF001 (unicode in user-facing strings).

21 ruff lint errors and 50 mypy errors remain for manual review.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Justin Visser 2026-02-26 22:11:41 +01:00
parent 399276ea69
commit bf04c77aa9
137 changed files with 3066 additions and 1900 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app/services/domains/arxiv/application.py
View file

@ -1,11 +1,11 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from app.services.domains.arxiv.gateway import (
build_arxiv_query,
get_arxiv_gateway,
)
from app.services.domains.arxiv.errors import ArxivRateLimitError
if TYPE_CHECKING:
from app.services.domains.publications.types import PublicationListItem, UnreadPublicationItem

64
app/services/domains/arxiv/cache.py
View file

@ -1,11 +1,11 @@
from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import asdict
from datetime import datetime, timedelta, timezone
import hashlib
import json
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import asdict
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, select
@ -33,15 +33,12 @@ async def get_cached_feed(
) -> ArxivFeed | None:
timestamp = _as_utc(now_utc)
session_factory = get_session_factory()
async with session_factory() as db_session:
async with db_session.begin():
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
)
)
entry = result.scalar_one_or_none()
return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
async with session_factory() as db_session, db_session.begin():
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
)
entry = result.scalar_one_or_none()
return await _validate_cached_entry(db_session, entry=entry, now_utc=timestamp)
async def set_cached_feed(
@ -54,16 +51,15 @@ async def set_cached_feed(
) -> None:
timestamp = _as_utc(now_utc)
session_factory = get_session_factory()
async with session_factory() as db_session:
async with db_session.begin():
await _write_cached_entry(
db_session,
query_fingerprint=query_fingerprint,
feed=feed,
ttl_seconds=ttl_seconds,
max_entries=max_entries,
now_utc=timestamp,
)
async with session_factory() as db_session, db_session.begin():
await _write_cached_entry(
db_session,
query_fingerprint=query_fingerprint,
feed=feed,
ttl_seconds=ttl_seconds,
max_entries=max_entries,
now_utc=timestamp,
)
async def run_with_inflight_dedupe(
@ -142,9 +138,7 @@ async def _write_cached_entry(
) -> None:
ttl = max(float(ttl_seconds), 0.0)
result = await db_session.execute(
select(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint == query_fingerprint
)
select(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint == query_fingerprint)
)
existing = result.scalar_one_or_none()
if ttl <= 0.0:
@ -177,30 +171,22 @@ async def _prune_cache_entries(
now_utc: datetime,
max_entries: int,
) -> None:
await db_session.execute(
delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc)
)
await db_session.execute(delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.expires_at <= now_utc))
bounded_max_entries = int(max_entries)
if bounded_max_entries <= 0:
return
count_result = await db_session.execute(
select(func.count()).select_from(ArxivQueryCacheEntry)
)
count_result = await db_session.execute(select(func.count()).select_from(ArxivQueryCacheEntry))
entry_count = int(count_result.scalar_one() or 0)
overflow = max(0, entry_count - bounded_max_entries)
if overflow <= 0:
return
stale_result = await db_session.execute(
select(ArxivQueryCacheEntry.query_fingerprint)
.order_by(ArxivQueryCacheEntry.cached_at.asc())
.limit(overflow)
select(ArxivQueryCacheEntry.query_fingerprint).order_by(ArxivQueryCacheEntry.cached_at.asc()).limit(overflow)
)
stale_keys = [str(row[0]) for row in stale_result.all()]
if stale_keys:
await db_session.execute(
delete(ArxivQueryCacheEntry).where(
ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys)
)
delete(ArxivQueryCacheEntry).where(ArxivQueryCacheEntry.query_fingerprint.in_(stale_keys))
)
@ -275,9 +261,9 @@ def _as_string_list(value: object) -> list[str]:
def _as_utc(value: datetime | None) -> datetime:
if value is None:
return datetime.now(timezone.utc)
return datetime.now(UTC)
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.replace(tzinfo=UTC)
return value

20
app/services/domains/arxiv/client.py
View file

@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Awaitable, Callable
import logging
from collections.abc import Awaitable, Callable
import httpx
@ -103,9 +103,13 @@ class ArxivClient:
if self._cache_enabled:
cached = await get_cached_feed(query_fingerprint=query_fingerprint)
if cached is not None:
structured_log(logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path)
structured_log(
logger, "info", "arxiv.cache_hit", query_fingerprint=query_fingerprint, source_path=source_path
)
return cached
structured_log(logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path)
structured_log(
logger, "info", "arxiv.cache_miss", query_fingerprint=query_fingerprint, source_path=source_path
)
return await run_with_inflight_dedupe(
query_fingerprint=query_fingerprint,
fetch_feed=lambda: self._fetch_live_feed(
@ -216,13 +220,13 @@ async def _request_arxiv_feed(
cooldown_status = await get_arxiv_cooldown_status()
if cooldown_status.is_active:
structured_log(
logger, "warning", "arxiv.request_skipped_cooldown",
logger,
"warning",
"arxiv.request_skipped_cooldown",
source_path=source_path,
cooldown_remaining_seconds=float(cooldown_status.remaining_seconds),
)
raise ArxivRateLimitError(
f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)"
)
raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_status.remaining_seconds:.0f}s remaining)")
async def _fetch() -> httpx.Response:
timeout_value = _timeout_seconds(timeout_seconds)
@ -272,5 +276,3 @@ def _source_path_from_params(params: dict[str, object]) -> str:
if "id_list" in params:
return ARXIV_SOURCE_PATH_LOOKUP_IDS
return ARXIV_SOURCE_PATH_UNKNOWN

2
app/services/domains/arxiv/gateway.py
View file

@ -2,8 +2,8 @@ from __future__ import annotations
import logging
import re
from typing import TYPE_CHECKING, Protocol
import unicodedata
from typing import TYPE_CHECKING, Protocol
from app.logging_utils import structured_log
from app.services.domains.arxiv.client import ArxivClient

52
app/services/domains/arxiv/rate_limit.py
View file

@ -1,10 +1,10 @@
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
import logging
from datetime import UTC, datetime, timedelta
import httpx
from sqlalchemy import select, text
@ -47,13 +47,11 @@ async def run_with_global_arxiv_limit(
async def get_arxiv_cooldown_status(*, now_utc: datetime | None = None) -> ArxivCooldownStatus:
timestamp = _normalize_datetime(now_utc) or datetime.now(timezone.utc)
timestamp = _normalize_datetime(now_utc) or datetime.now(UTC)
session_factory = get_session_factory()
async with session_factory() as db_session:
result = await db_session.execute(
select(ArxivRuntimeState.cooldown_until).where(
ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY
)
select(ArxivRuntimeState.cooldown_until).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
)
cooldown_until = _normalize_datetime(result.scalar_one_or_none())
remaining_seconds = _cooldown_remaining_seconds(cooldown_until, now_utc=timestamp)
@ -85,10 +83,14 @@ async def _run_serialized_fetch(
source_path=source_path,
)
structured_log(
logger, "info", "arxiv.request_completed",
logger,
"info",
"arxiv.request_completed",
status_code=int(response.status_code),
wait_seconds=wait_seconds,
cooldown_remaining_seconds=_cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=datetime.now(timezone.utc)),
cooldown_remaining_seconds=_cooldown_remaining_seconds(
runtime_state.cooldown_until, now_utc=datetime.now(UTC)
),
source_path=source_path,
)
return response, hit_rate_limit
@ -106,9 +108,7 @@ async def _acquire_arxiv_lock(db_session: AsyncSession) -> None:
async def _load_runtime_state_for_update(db_session: AsyncSession) -> ArxivRuntimeState:
result = await db_session.execute(
select(ArxivRuntimeState)
.where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY)
.with_for_update()
select(ArxivRuntimeState).where(ArxivRuntimeState.state_key == ARXIV_RUNTIME_STATE_KEY).with_for_update()
)
state = result.scalar_one_or_none()
if state is not None:
@ -124,13 +124,27 @@ async def _wait_for_allowed_slot_or_raise(
*,
source_path: str,
) -> float:
now_utc = datetime.now(timezone.utc)
now_utc = datetime.now(UTC)
cooldown_seconds = _cooldown_remaining_seconds(runtime_state.cooldown_until, now_utc=now_utc)
if cooldown_seconds > 0:
structured_log(logger, "info", "arxiv.request_scheduled", wait_seconds=0.0, source_path=source_path, cooldown_remaining_seconds=cooldown_seconds)
structured_log(
logger,
"info",
"arxiv.request_scheduled",
wait_seconds=0.0,
source_path=source_path,
cooldown_remaining_seconds=cooldown_seconds,
)
raise ArxivRateLimitError(f"arXiv global cooldown active ({cooldown_seconds:.0f}s remaining)")
wait_seconds = _next_allowed_wait_seconds(runtime_state.next_allowed_at, now_utc=now_utc)
structured_log(logger, "info", "arxiv.request_scheduled", wait_seconds=wait_seconds, source_path=source_path, cooldown_remaining_seconds=0.0)
structured_log(
logger,
"info",
"arxiv.request_scheduled",
wait_seconds=wait_seconds,
source_path=source_path,
cooldown_remaining_seconds=0.0,
)
if wait_seconds > 0:
await asyncio.sleep(wait_seconds)
return wait_seconds
@ -142,13 +156,15 @@ def _record_post_response_state(
response_status: int,
source_path: str,
) -> bool:
now_utc = datetime.now(timezone.utc)
now_utc = datetime.now(UTC)
runtime_state.next_allowed_at = now_utc + timedelta(seconds=_min_interval_seconds())
if response_status == 429:
cooldown_seconds = _cooldown_seconds()
runtime_state.cooldown_until = now_utc + timedelta(seconds=cooldown_seconds)
structured_log(
logger, "warning", "arxiv.cooldown_activated",
logger,
"warning",
"arxiv.cooldown_activated",
cooldown_remaining_seconds=cooldown_seconds,
source_path=source_path,
)
@ -176,7 +192,7 @@ def _normalize_datetime(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.replace(tzinfo=UTC)
return value
@ -186,5 +202,3 @@ def _min_interval_seconds() -> float:
def _cooldown_seconds() -> float:
return max(float(settings.arxiv_rate_limit_cooldown_seconds), 0.0)